{
 "cells": [
  {
   "cell_type": "code",
   "id": "c0f56bed-43b5-4030-a9ed-420bec32c605",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:43:44.599898Z",
     "iopub.status.busy": "2024-08-18T14:43:44.598835Z",
     "iopub.status.idle": "2024-08-18T14:43:44.611174Z",
     "shell.execute_reply": "2024-08-18T14:43:44.610542Z",
     "shell.execute_reply.started": "2024-08-18T14:43:44.599817Z"
    },
    "ExecuteTime": {
     "end_time": "2024-08-20T15:17:28.633152Z",
     "start_time": "2024-08-20T15:17:28.620052Z"
    }
   },
   "source": [
    "%env LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1\n",
    "%env LLM_API_KEY=替换为自己的key"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1\n",
      "env: LLM_API_KEY=替换为自己的key\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dc3d95d2-96ce-46b4-a112-8b1d1c951407",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:43:44.612228Z",
     "iopub.status.busy": "2024-08-18T14:43:44.612013Z",
     "iopub.status.idle": "2024-08-18T14:43:44.618665Z",
     "shell.execute_reply": "2024-08-18T14:43:44.617741Z",
     "shell.execute_reply.started": "2024-08-18T14:43:44.612214Z"
    }
   },
   "outputs": [],
   "source": [
    "%%capture --no-stderr\n",
    "!pip install -U langchain langchain_community langchain_openai pypdf sentence_transformers chromadb shutil openpyxl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1e2c72b8-ee12-4130-af88-699998aa230c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:43:44.619426Z",
     "iopub.status.busy": "2024-08-18T14:43:44.619262Z",
     "iopub.status.idle": "2024-08-18T14:43:48.840614Z",
     "shell.execute_reply": "2024-08-18T14:43:48.840140Z",
     "shell.execute_reply.started": "2024-08-18T14:43:44.619413Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.10/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:11: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm, trange\n",
      "2024-08-18 22:43:46.467817: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2024-08-18 22:43:46.499171: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-08-18 22:43:47.095997: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "langchain                     0.2.10\n",
      "langchain_core                0.2.28\n",
      "langchain_community           0.2.9\n",
      "pypdf                         4.3.1\n",
      "sentence_transformers         3.0.1\n",
      "chromadb                      0.5.4\n"
     ]
    }
   ],
   "source": [
    "import langchain, langchain_community, pypdf, sentence_transformers, chromadb, langchain_core\n",
    "\n",
    "for module in (langchain, langchain_core, langchain_community, pypdf, sentence_transformers, chromadb):\n",
    "    print(f\"{module.__name__:<30}{module.__version__}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c2854bf0-5eee-4e73-be29-48421b7f024f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:43:48.841401Z",
     "iopub.status.busy": "2024-08-18T14:43:48.841114Z",
     "iopub.status.idle": "2024-08-18T14:43:48.843677Z",
     "shell.execute_reply": "2024-08-18T14:43:48.843254Z",
     "shell.execute_reply.started": "2024-08-18T14:43:48.841387Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "841d2b02-ad06-40d2-b11f-c7adccec6ca2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:43:48.844884Z",
     "iopub.status.busy": "2024-08-18T14:43:48.844756Z",
     "iopub.status.idle": "2024-08-18T14:43:48.856923Z",
     "shell.execute_reply": "2024-08-18T14:43:48.856546Z",
     "shell.execute_reply.started": "2024-08-18T14:43:48.844871Z"
    }
   },
   "outputs": [],
   "source": [
    "expr_version = 'retrieval_v3_rag_fusion'\n",
    "\n",
    "preprocess_output_dir = os.path.join(os.path.pardir, 'outputs', 'v1_20240713')\n",
    "expr_dir = os.path.join(os.path.pardir, 'experiments', expr_version)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf7e81e3-4c82-4842-aef5-7592caaf1d39",
   "metadata": {},
   "source": [
    "# 读取文档"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "da15f02e-3131-43fb-81c5-f4da615c449b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:43:48.857618Z",
     "iopub.status.busy": "2024-08-18T14:43:48.857445Z",
     "iopub.status.idle": "2024-08-18T14:43:50.332650Z",
     "shell.execute_reply": "2024-08-18T14:43:50.332171Z",
     "shell.execute_reply.started": "2024-08-18T14:43:48.857605Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import PyPDFLoader\n",
    "\n",
    "loader = PyPDFLoader(os.path.join(os.path.pardir, 'data', '2024全球经济金融展望报告.pdf'))\n",
    "documents = loader.load()\n",
    "\n",
    "qa_df = pd.read_excel(os.path.join(preprocess_output_dir, 'question_answer.xlsx'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "841ec659-4ad7-4e1f-b1ea-3477bf97fde3",
   "metadata": {},
   "source": [
    "# 文档切分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "74fe856a-7c19-4c3c-bb30-7abfa6298f74",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:43:50.333292Z",
     "iopub.status.busy": "2024-08-18T14:43:50.333052Z",
     "iopub.status.idle": "2024-08-18T14:43:50.339605Z",
     "shell.execute_reply": "2024-08-18T14:43:50.339220Z",
     "shell.execute_reply.started": "2024-08-18T14:43:50.333277Z"
    }
   },
   "outputs": [],
   "source": [
    "from uuid import uuid4\n",
    "import os\n",
    "import pickle\n",
    "\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "def split_docs(documents, filepath, chunk_size=400, chunk_overlap=40, seperators=['\\n\\n\\n', '\\n\\n'], force_split=False):\n",
    "    if os.path.exists(filepath) and not force_split:\n",
    "        print('found cache, restoring...')\n",
    "        return pickle.load(open(filepath, 'rb'))\n",
    "\n",
    "    splitter = RecursiveCharacterTextSplitter(\n",
    "        chunk_size=chunk_size,\n",
    "        chunk_overlap=chunk_overlap,\n",
    "        separators=seperators\n",
    "    )\n",
    "    split_docs = splitter.split_documents(documents)\n",
    "    for chunk in split_docs:\n",
    "        chunk.metadata['uuid'] = str(uuid4())\n",
    "\n",
    "    pickle.dump(split_docs, open(filepath, 'wb'))\n",
    "\n",
    "    return split_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "aa25540d-0504-4ae7-9804-9e3862b132d5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:43:50.340300Z",
     "iopub.status.busy": "2024-08-18T14:43:50.340173Z",
     "iopub.status.idle": "2024-08-18T14:43:50.351146Z",
     "shell.execute_reply": "2024-08-18T14:43:50.350640Z",
     "shell.execute_reply.started": "2024-08-18T14:43:50.340287Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found cache, restoring...\n"
     ]
    }
   ],
   "source": [
    "splitted_docs = split_docs(documents, os.path.join(preprocess_output_dir, 'split_docs.pkl'), chunk_size=500, chunk_overlap=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "220dbc3a-fceb-4e49-a3f1-01e16660b2a6",
   "metadata": {},
   "source": [
    "# 检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8598a11c-25d8-4af1-a98b-06a8c394e261",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:43:50.351785Z",
     "iopub.status.busy": "2024-08-18T14:43:50.351624Z",
     "iopub.status.idle": "2024-08-18T14:43:50.367378Z",
     "shell.execute_reply": "2024-08-18T14:43:50.366869Z",
     "shell.execute_reply.started": "2024-08-18T14:43:50.351772Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "device: cuda\n"
     ]
    }
   ],
   "source": [
    "from langchain.embeddings import HuggingFaceBgeEmbeddings\n",
    "from langchain_community.vectorstores import Chroma\n",
    "import torch\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "print(f'device: {device}')\n",
    "\n",
    "def get_embeddings(model_path):\n",
    "    embeddings = HuggingFaceBgeEmbeddings(\n",
    "        model_name=model_path,\n",
    "        model_kwargs={'device': device},\n",
    "        encode_kwargs={'normalize_embeddings': True},\n",
    "        # show_progress=True\n",
    "        query_instruction='为这个句子生成表示以用于检索相关文章：'\n",
    "    )\n",
    "    return embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "663ef1a4-5866-4f6b-8d9d-4724f62142cb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:43:50.367973Z",
     "iopub.status.busy": "2024-08-18T14:43:50.367843Z",
     "iopub.status.idle": "2024-08-18T14:44:04.795337Z",
     "shell.execute_reply": "2024-08-18T14:44:04.792820Z",
     "shell.execute_reply.started": "2024-08-18T14:43:50.367960Z"
    }
   },
   "outputs": [],
   "source": [
    "import shutil\n",
    "from langchain_community.vectorstores import Chroma\n",
    "\n",
    "model_path = 'BAAI/bge-large-zh-v1.5'\n",
    "\n",
    "persist_directory = os.path.join(expr_dir, 'chroma')\n",
    "shutil.rmtree(persist_directory, ignore_errors=True)\n",
    "\n",
    "embeddings = get_embeddings(model_path)\n",
    "vector_db = Chroma.from_documents(\n",
    "    splitted_docs,\n",
    "    embedding=embeddings,\n",
    "    persist_directory=persist_directory\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b03e3382-39e9-4932-a265-69b811041629",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:44:04.798443Z",
     "iopub.status.busy": "2024-08-18T14:44:04.797720Z",
     "iopub.status.idle": "2024-08-18T14:44:04.811935Z",
     "shell.execute_reply": "2024-08-18T14:44:04.809504Z",
     "shell.execute_reply.started": "2024-08-18T14:44:04.798373Z"
    }
   },
   "outputs": [],
   "source": [
    "test_df = qa_df[(qa_df['dataset'] == 'test') & (qa_df['qa_type'] == 'detailed')]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b2e4b81-bd77-45e7-bfb0-ae321e51fe90",
   "metadata": {},
   "source": [
    "## 不使用RAG Fusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d448410f-ca18-4eb3-b9f2-33ef3656b796",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:44:04.813235Z",
     "iopub.status.busy": "2024-08-18T14:44:04.813107Z",
     "iopub.status.idle": "2024-08-18T14:44:04.825635Z",
     "shell.execute_reply": "2024-08-18T14:44:04.823336Z",
     "shell.execute_reply.started": "2024-08-18T14:44:04.813223Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_emb_retriever(k):\n",
    "    return vector_db.as_retriever(search_kwargs={'k': k})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "32c3ad14-b217-44aa-bdb9-909b9d559668",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:44:04.828308Z",
     "iopub.status.busy": "2024-08-18T14:44:04.827594Z",
     "iopub.status.idle": "2024-08-18T14:44:04.846861Z",
     "shell.execute_reply": "2024-08-18T14:44:04.844485Z",
     "shell.execute_reply.started": "2024-08-18T14:44:04.828240Z"
    }
   },
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n",
    "\n",
    "def get_hit_stat_df(get_retriever_fn, top_k_arr=list(range(1, 9))):\n",
    "    hit_stat_data = []\n",
    "    pbar = tqdm(total=len(top_k_arr) * len(test_df))\n",
    "    for k in top_k_arr:\n",
    "        pbar.set_description(f'k={k}')\n",
    "        retriever = get_retriever_fn(k)\n",
    "        \n",
    "        for idx, row in test_df.iterrows():\n",
    "            question = row['question']\n",
    "            true_uuid = row['uuid']\n",
    "            \n",
    "            chunks = retriever.invoke(question)[:k]\n",
    "            retrieved_uuids = [doc.metadata['uuid'] for doc in chunks]\n",
    "\n",
    "            hit_stat_data.append({\n",
    "                'question': question,\n",
    "                'top_k': k,\n",
    "                'hit': int(true_uuid in retrieved_uuids),\n",
    "                'retrieved_chunks': len(chunks)\n",
    "            })\n",
    "            pbar.update(1)\n",
    "    hit_stat_df = pd.DataFrame(hit_stat_data)\n",
    "    return hit_stat_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "93dd701a-bd89-44db-a954-c78dd7001e38",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:44:04.859519Z",
     "iopub.status.busy": "2024-08-18T14:44:04.858002Z",
     "iopub.status.idle": "2024-08-18T14:44:22.499963Z",
     "shell.execute_reply": "2024-08-18T14:44:22.499487Z",
     "shell.execute_reply.started": "2024-08-18T14:44:04.859435Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d1482a4279f4fa8a34da042bf01f073",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/744 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "orig_query_hit_stat_df = get_hit_stat_df(get_emb_retriever)\n",
    "orig_query_hit_stat_df['rag_fusion'] = 'w/o'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd9f5f38-28b2-4b77-a101-8694e505678f",
   "metadata": {},
   "source": [
    "## 使用RAG Fusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f25c0878-151a-4d06-901d-3bf4add8a86a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:44:22.500565Z",
     "iopub.status.busy": "2024-08-18T14:44:22.500434Z",
     "iopub.status.idle": "2024-08-18T14:44:22.565835Z",
     "shell.execute_reply": "2024-08-18T14:44:22.565356Z",
     "shell.execute_reply.started": "2024-08-18T14:44:22.500551Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain_community.chat_models import ChatOllama\n",
    "from langchain.prompts import PromptTemplate\n",
    "import re\n",
    "\n",
    "llm = ChatOllama(base_url='http://localhost:11434', model='qwen2:7b-instruct')\n",
    "\n",
    "prompt = PromptTemplate(\n",
    "    input_variables=['question', 'n_sim_query'],\n",
    "    template = \"\"\"你是一个AI语言模型助手。你的任务是基于给定的原始问题，再生成出来最相似的{n_sim_query}个不同的版本。\\\n",
    "你的目标是通过生成用户问题不同视角的版本，帮助用户克服基于距离做相似性查找的局限性。\\\n",
    "使用换行符来提供这些不同的问题，使用换行符来切分不同的问题，不要包含数字序号，仅返回结果即可，不要添加任何其他描述性文本。\n",
    "原始问题：{question}\n",
    "\"\"\"\n",
    ")\n",
    "\n",
    "generate_queries_chain = (\n",
    "    prompt\n",
    "    | llm\n",
    "    # 有时候模型不遵循指令，把前面的序号去掉\n",
    "    # | (lambda x: re.sub(r'\\d+\\.\\s', '', x.content))\n",
    "    # 有时候模型不遵循指令，把前面的序号、- 去掉\n",
    "    | (lambda x: [\n",
    "        re.sub(r'(^\\-\\s+)|(^\\d+\\.\\s)', '', item.strip()) \n",
    "        for item in x.content.split('\\n') if item.strip() != ''\n",
    "    ])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a81f2a65-ab2a-4a02-b35a-c6d3ff384479",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:44:22.566489Z",
     "iopub.status.busy": "2024-08-18T14:44:22.566352Z",
     "iopub.status.idle": "2024-08-18T14:44:22.569953Z",
     "shell.execute_reply": "2024-08-18T14:44:22.569607Z",
     "shell.execute_reply.started": "2024-08-18T14:44:22.566476Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain.load import dumps, loads\n",
    "\n",
    "def reciprocal_rank_fusion(results: list[list], k=60):\n",
    "    fused_scores = {}\n",
    "    for docs in results:\n",
    "        # 此处有一个隐含的假设：返回的docs是按相似度排好序的\n",
    "        for rank, doc in enumerate(docs):\n",
    "            doc_str = dumps(doc)\n",
    "            if doc_str not in fused_scores:\n",
    "                fused_scores[doc_str] = 0\n",
    "            # previous_score = fused_scores[doc_str]\n",
    "            fused_scores[doc_str] += 1 / (rank + k)\n",
    "    reranked_results = [\n",
    "        (loads(doc), score)\n",
    "        for doc, score in sorted(fused_scores.items(), key=lambda x: x[1], reverse=True)\n",
    "    ]\n",
    "    return reranked_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "49ccacb6-52b0-4b97-9305-27fb801fefdb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:44:22.570504Z",
     "iopub.status.busy": "2024-08-18T14:44:22.570382Z",
     "iopub.status.idle": "2024-08-18T14:44:22.577476Z",
     "shell.execute_reply": "2024-08-18T14:44:22.577099Z",
     "shell.execute_reply.started": "2024-08-18T14:44:22.570492Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_rag_fusion_chain(top_k, n_sim_query=3, trunc=False):\n",
    "    \"\"\"\n",
    "    获取RAG Fusion Chain\n",
    "    :param top_k: 每个相似问检索的片段数量\n",
    "    :param trunc: 最终排序后的结果是否要截断为top_k\n",
    "    \"\"\"\n",
    "    chain = (\n",
    "        prompt.partial(n_sim_query=n_sim_query)\n",
    "        | llm\n",
    "        # 有时候模型不遵循指令，把前面的序号去掉\n",
    "        # | (lambda x: re.sub(r'\\d+\\.\\s', '', x.content))\n",
    "        # 有时候模型不遵循指令，把前面的序号、- 去掉\n",
    "        | (lambda x: [\n",
    "            re.sub(r'(^\\-\\s+)|(^\\d+\\.\\s)', '', item.strip()) \n",
    "            for item in x.content.split('\\n') if item.strip() != ''])\n",
    "        | (lambda x: [vector_db.similarity_search(q, k=top_k) for q in x])\n",
    "        | reciprocal_rank_fusion\n",
    "        | (lambda docs: docs[:top_k] if trunc else docs)\n",
    "    )\n",
    "    return chain"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5092acc2-44ed-495a-9f7a-051a5f5876ea",
   "metadata": {},
   "source": [
    "首先先生成相似问题，这个跟Multi Query是类似的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "67af90ea-03af-473e-a914-f6a2ea2f02e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:44:22.578003Z",
     "iopub.status.busy": "2024-08-18T14:44:22.577878Z",
     "iopub.status.idle": "2024-08-18T14:44:23.047510Z",
     "shell.execute_reply": "2024-08-18T14:44:23.047026Z",
     "shell.execute_reply.started": "2024-08-18T14:44:22.577991Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['报告是由哪个组织发布的？', '谁能找到报告的来源？', '什么实体负责公布这份报告？']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate_queries_chain.invoke({'question': '报告的发布机构是什么？', 'n_sim_query': 3})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c52b9c5-b703-4c80-828c-19f024a79b6d",
   "metadata": {},
   "source": [
    "查看一下中间过程，这个步骤，对于每一个产生的相似问题，检索5个片段"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "15ca104a-99a3-48df-b633-30c6816fe924",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:44:23.048112Z",
     "iopub.status.busy": "2024-08-18T14:44:23.047967Z",
     "iopub.status.idle": "2024-08-18T14:44:23.538203Z",
     "shell.execute_reply": "2024-08-18T14:44:23.537719Z",
     "shell.execute_reply.started": "2024-08-18T14:44:23.048099Z"
    }
   },
   "outputs": [],
   "source": [
    "retrieved_docs = (\n",
    "    generate_queries_chain\n",
    "    | (lambda x: [vector_db.similarity_search(q, k=5) for q in x])\n",
    ").invoke({'question': '报告的发布机构是什么？', 'n_sim_query': 3})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "84dadabf-e725-4836-8557-d081cea4430a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:44:23.538889Z",
     "iopub.status.busy": "2024-08-18T14:44:23.538708Z",
     "iopub.status.idle": "2024-08-18T14:44:23.541850Z",
     "shell.execute_reply": "2024-08-18T14:44:23.541529Z",
     "shell.execute_reply.started": "2024-08-18T14:44:23.538875Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(retrieved_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "96b22543-b471-4379-9390-b842cb65baea",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:44:23.542390Z",
     "iopub.status.busy": "2024-08-18T14:44:23.542270Z",
     "iopub.status.idle": "2024-08-18T14:44:23.562886Z",
     "shell.execute_reply": "2024-08-18T14:44:23.560546Z",
     "shell.execute_reply.started": "2024-08-18T14:44:23.542378Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(retrieved_docs[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d3cf525f-0ca4-478c-a37a-00fbc1f91928",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:44:23.566421Z",
     "iopub.status.busy": "2024-08-18T14:44:23.565691Z",
     "iopub.status.idle": "2024-08-18T14:44:24.215777Z",
     "shell.execute_reply": "2024-08-18T14:44:24.215290Z",
     "shell.execute_reply.started": "2024-08-18T14:44:23.566351Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.10/site-packages/langchain_core/_api/beta_decorator.py:87: LangChainBetaWarning: The function `loads` is in beta. It is actively being worked on, so the API may change.\n",
      "  warn_beta(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(get_rag_fusion_chain(3).invoke({'question': '报告的发布机构是什么？'}))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06d0d592-4644-40a3-87e0-8b867b4487c4",
   "metadata": {},
   "source": [
    "如果确认rag_fusion_chain没有问题，可以直接设置trunc为True，确保召回数量符合设定值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c87526ad-3788-48b2-bff7-6a2435a5241b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:44:24.216393Z",
     "iopub.status.busy": "2024-08-18T14:44:24.216268Z",
     "iopub.status.idle": "2024-08-18T14:44:24.682977Z",
     "shell.execute_reply": "2024-08-18T14:44:24.682582Z",
     "shell.execute_reply.started": "2024-08-18T14:44:24.216380Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(Document(metadata={'page': 33, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': '3e312dfa-dd43-4ab9-961f-a2e442c89cdd'}, page_content='全球经济金融展望报告\\n中国银行研究院 32 2024年\\n图19：美国联邦基金目标利率与全球MSCI指数\\n资料来源：Wind，中国银行研究院\\n表3：全球主要股指概览\\n注：涨跌幅区间为2023年1月1日至2023年11月15日，收盘价和市盈\\n率为2023年11月15日。\\n资料来源：Wind，中国银行研究院'),\n",
       "  0.04972677595628415),\n",
       " (Document(metadata={'page': 51, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': 'ebf0d999-59f6-4fd3-941e-05a7a60c255a'}, page_content='免责声明\\n本研究报告由中国银行研究院撰写，研究报告中所引用信息均来自公开资料。\\n本研究报告中包含的观点或估计仅代表作者迄今为止的判断，它们不一定反映中国银行的观点。中国\\n银行研究院可以不经通知加以改变，且没有对此报告更新、修正或修改的责任。\\n本研究报告内容及观点仅供参考，不构成任何投资建议。对于本报告所提供信息所导致的任何直接的\\n或者间接的投资盈亏后果不承担任何责任。\\n本研究报告版权仅为中国银行研究院所有，未经书面许可，任何机构和个人不得以任何形式翻版、复\\n制和发布。如引用发布，需注明出处为中国银行研究院，且不得对本报告进行有悖原意的引用、删节和修\\n改。中国银行研究院保留对任何侵权行为和有悖报告原意的引用行为进行追究的权利。'),\n",
       "  0.03278688524590164)]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_rag_fusion_chain(2, trunc=True).invoke({'question': '报告的发布机构是什么？'})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aabd3d3e-4cc6-4a6d-bdf9-22e15d0bf2d8",
   "metadata": {},
   "source": [
    "为了方便调参，我们创建一个新的函数，使用下面这个函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "cc4076de-a180-4026-aa64-83fdd2fb0005",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:44:24.683552Z",
     "iopub.status.busy": "2024-08-18T14:44:24.683424Z",
     "iopub.status.idle": "2024-08-18T14:44:24.686760Z",
     "shell.execute_reply": "2024-08-18T14:44:24.686424Z",
     "shell.execute_reply.started": "2024-08-18T14:44:24.683539Z"
    }
   },
   "outputs": [],
   "source": [
    "def retrieve_with_rrf(llm, query, top_k=4, n_sim_query=3, include_original=True, trunc=True):\n",
    "    \"\"\"\n",
    "    使用RRF检索\n",
    "    :param llm: 用于生成相似问题的LLM\n",
    "    :param query: 需要检索的问题\n",
    "    :param top_k: 每个问题检索几个知识片段\n",
    "    :param n_sim_query: 每个query生成几个相似问题\n",
    "    :param include_original: 检索知识片段时，是否包含原始问题\n",
    "    :param trunc: 是否将最终检索结果，截断为top_k\n",
    "    \"\"\"\n",
    "    chain = (\n",
    "        prompt.partial(n_sim_query=n_sim_query)\n",
    "        | llm\n",
    "        # 有时候模型不遵循指令，把前面的序号、- 去掉\n",
    "        | (lambda x: [\n",
    "            re.sub(r'(^\\-\\s+)|(^\\d+\\.\\s)', '', item.strip()) \n",
    "            for item in x.content.split('\\n') if item.strip() != ''])\n",
    "        | (lambda x: [vector_db.similarity_search(q, k=top_k) for q in ([query] if include_original else []) + x[:n_sim_query]])\n",
    "        | reciprocal_rank_fusion\n",
    "        | (lambda docs: docs[:top_k] if trunc else docs)\n",
    "    )\n",
    "    return chain.invoke(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "5dfbbf07-8218-47e4-a8b8-557f56026b69",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:44:24.687304Z",
     "iopub.status.busy": "2024-08-18T14:44:24.687179Z",
     "iopub.status.idle": "2024-08-18T14:44:24.707414Z",
     "shell.execute_reply": "2024-08-18T14:44:24.705034Z",
     "shell.execute_reply.started": "2024-08-18T14:44:24.687292Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_rag_fusion_hit_stat_df(query_gen_llm, top_k_arr=list(range(1, 9))):\n",
    "    hit_stat_data = []\n",
    "    pbar = tqdm(total=len(top_k_arr) * len(test_df))\n",
    "    for k in top_k_arr:\n",
    "        pbar.set_description(f'k={k}')\n",
    "        rag_fusion_chain = get_rag_fusion_chain(k, trunc=True)\n",
    "        for idx, row in test_df.iterrows():\n",
    "            question = row['question']\n",
    "            true_uuid = row['uuid']\n",
    "            # chunks = rag_fusion_chain.invoke({'question': question})\n",
    "            chunks = retrieve_with_rrf(query_gen_llm, question, top_k=k)\n",
    "            assert len(chunks) <= k\n",
    "            \n",
    "            retrieved_uuids = [doc.metadata['uuid'] for doc, score in chunks]\n",
    "\n",
    "            hit_stat_data.append({\n",
    "                'question': question,\n",
    "                'top_k': k,\n",
    "                'hit': int(true_uuid in retrieved_uuids),\n",
    "                'retrieved_chunks': len(chunks)\n",
    "            })\n",
    "            pbar.update(1)\n",
    "    hit_stat_df = pd.DataFrame(hit_stat_data)\n",
    "    return hit_stat_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a9c8ee12-b922-42bf-b80a-283cf65ee1b2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T14:44:24.710421Z",
     "iopub.status.busy": "2024-08-18T14:44:24.709706Z",
     "iopub.status.idle": "2024-08-18T15:23:05.972080Z",
     "shell.execute_reply": "2024-08-18T15:23:05.971733Z",
     "shell.execute_reply.started": "2024-08-18T14:44:24.710351Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c31066017d92489181dc2990dffa8274",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/744 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "477f512b537b42678d8476acf869b661",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/744 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "qwen2_14b_llm = ChatOpenAI(\n",
    "    base_url=os.environ['LLM_BASE_URL'],\n",
    "    api_key=os.environ['LLM_API_KEY'],\n",
    "    model='qwen2-57b-a14b-instruct'\n",
    ")\n",
    "\n",
    "rag_fusion_hit_stat_dfs = []\n",
    "for llm_name, llm_model in zip(['ollama-qwen2-7b-instruct', 'qwen2-57b-a14b-instruct'], [llm, qwen2_14b_llm]):\n",
    "    rag_fusion_hit_stat_df = get_rag_fusion_hit_stat_df(llm_model)\n",
    "    rag_fusion_hit_stat_df['rag_fusion'] = f'w/ {llm_name}'\n",
    "    rag_fusion_hit_stat_dfs.append(rag_fusion_hit_stat_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "899d1393-3ba3-48f7-b10e-c5c3de8a8927",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T15:23:05.972669Z",
     "iopub.status.busy": "2024-08-18T15:23:05.972541Z",
     "iopub.status.idle": "2024-08-18T15:23:05.975285Z",
     "shell.execute_reply": "2024-08-18T15:23:05.974922Z",
     "shell.execute_reply.started": "2024-08-18T15:23:05.972655Z"
    }
   },
   "outputs": [],
   "source": [
    "hit_stat_df = pd.concat([orig_query_hit_stat_df] + rag_fusion_hit_stat_dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d4890789-a44c-41de-b17f-0ff505788494",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T15:23:05.975814Z",
     "iopub.status.busy": "2024-08-18T15:23:05.975690Z",
     "iopub.status.idle": "2024-08-18T15:23:06.009634Z",
     "shell.execute_reply": "2024-08-18T15:23:06.009148Z",
     "shell.execute_reply.started": "2024-08-18T15:23:05.975802Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rag_fusion</th>\n",
       "      <th>top_k</th>\n",
       "      <th>hit_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>w/ ollama-qwen2-7b-instruct</td>\n",
       "      <td>1</td>\n",
       "      <td>0.451613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>w/ ollama-qwen2-7b-instruct</td>\n",
       "      <td>2</td>\n",
       "      <td>0.591398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>w/ ollama-qwen2-7b-instruct</td>\n",
       "      <td>3</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>w/ ollama-qwen2-7b-instruct</td>\n",
       "      <td>4</td>\n",
       "      <td>0.741935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>w/ ollama-qwen2-7b-instruct</td>\n",
       "      <td>5</td>\n",
       "      <td>0.806452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>w/ ollama-qwen2-7b-instruct</td>\n",
       "      <td>6</td>\n",
       "      <td>0.784946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>w/ ollama-qwen2-7b-instruct</td>\n",
       "      <td>7</td>\n",
       "      <td>0.817204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>w/ ollama-qwen2-7b-instruct</td>\n",
       "      <td>8</td>\n",
       "      <td>0.838710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>w/ qwen2-57b-a14b-instruct</td>\n",
       "      <td>1</td>\n",
       "      <td>0.462366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>w/ qwen2-57b-a14b-instruct</td>\n",
       "      <td>2</td>\n",
       "      <td>0.580645</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>w/ qwen2-57b-a14b-instruct</td>\n",
       "      <td>3</td>\n",
       "      <td>0.698925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>w/ qwen2-57b-a14b-instruct</td>\n",
       "      <td>4</td>\n",
       "      <td>0.774194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>w/ qwen2-57b-a14b-instruct</td>\n",
       "      <td>5</td>\n",
       "      <td>0.784946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>w/ qwen2-57b-a14b-instruct</td>\n",
       "      <td>6</td>\n",
       "      <td>0.806452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>w/ qwen2-57b-a14b-instruct</td>\n",
       "      <td>7</td>\n",
       "      <td>0.827957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>w/ qwen2-57b-a14b-instruct</td>\n",
       "      <td>8</td>\n",
       "      <td>0.849462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>w/o</td>\n",
       "      <td>1</td>\n",
       "      <td>0.462366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>w/o</td>\n",
       "      <td>2</td>\n",
       "      <td>0.591398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>w/o</td>\n",
       "      <td>3</td>\n",
       "      <td>0.688172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>w/o</td>\n",
       "      <td>4</td>\n",
       "      <td>0.774194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>w/o</td>\n",
       "      <td>5</td>\n",
       "      <td>0.806452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>w/o</td>\n",
       "      <td>6</td>\n",
       "      <td>0.817204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>w/o</td>\n",
       "      <td>7</td>\n",
       "      <td>0.838710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>w/o</td>\n",
       "      <td>8</td>\n",
       "      <td>0.849462</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     rag_fusion  top_k  hit_rate\n",
       "0   w/ ollama-qwen2-7b-instruct      1  0.451613\n",
       "1   w/ ollama-qwen2-7b-instruct      2  0.591398\n",
       "2   w/ ollama-qwen2-7b-instruct      3  0.666667\n",
       "3   w/ ollama-qwen2-7b-instruct      4  0.741935\n",
       "4   w/ ollama-qwen2-7b-instruct      5  0.806452\n",
       "5   w/ ollama-qwen2-7b-instruct      6  0.784946\n",
       "6   w/ ollama-qwen2-7b-instruct      7  0.817204\n",
       "7   w/ ollama-qwen2-7b-instruct      8  0.838710\n",
       "8    w/ qwen2-57b-a14b-instruct      1  0.462366\n",
       "9    w/ qwen2-57b-a14b-instruct      2  0.580645\n",
       "10   w/ qwen2-57b-a14b-instruct      3  0.698925\n",
       "11   w/ qwen2-57b-a14b-instruct      4  0.774194\n",
       "12   w/ qwen2-57b-a14b-instruct      5  0.784946\n",
       "13   w/ qwen2-57b-a14b-instruct      6  0.806452\n",
       "14   w/ qwen2-57b-a14b-instruct      7  0.827957\n",
       "15   w/ qwen2-57b-a14b-instruct      8  0.849462\n",
       "16                          w/o      1  0.462366\n",
       "17                          w/o      2  0.591398\n",
       "18                          w/o      3  0.688172\n",
       "19                          w/o      4  0.774194\n",
       "20                          w/o      5  0.806452\n",
       "21                          w/o      6  0.817204\n",
       "22                          w/o      7  0.838710\n",
       "23                          w/o      8  0.849462"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hit_stat_df.groupby(['rag_fusion', 'top_k'])['hit'].mean().reset_index().rename(columns={'hit': 'hit_rate'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b0b086d1-6cec-4743-8df6-2ab3b1593689",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T15:23:06.010365Z",
     "iopub.status.busy": "2024-08-18T15:23:06.010109Z",
     "iopub.status.idle": "2024-08-18T15:23:06.587331Z",
     "shell.execute_reply": "2024-08-18T15:23:06.586829Z",
     "shell.execute_reply.started": "2024-08-18T15:23:06.010351Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='top_k', ylabel='hit'>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGxCAYAAACeKZf2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAABM8ElEQVR4nO3de1zO9/8/8MdVOpeSjlrk2IEOVitp1IiYGbNNMxON9nFoDs1xQ85lSE7ThtRnY/gMYxs5XHM1p4nC19kYMjowVELluq7fH369uVQqHd5597jfbtft1vW+Xu/3+/lq43p4vV7v91umVqvVICIiIpIILbELICIiIqpODDdEREQkKQw3REREJCkMN0RERCQpDDdEREQkKQw3REREJCkMN0RERCQpDDdEREQkKQ3ELqC2qVQq3Lx5EyYmJpDJZGKXQ0RERBWgVquRl5eHJk2aQEvrxWMz9S7c3Lx5E/b29mKXQURERC/h+vXreO21117Ypt6FGxMTEwBPfjkNGzYUuRoiIiKqiNzcXNjb2wvf4y9S78JN8VRUw4YNGW6IiIheMRVZUsIFxURERCQpDDdEREQkKQw3REREJCn1bs1NRSmVShQVFYldBhFVEx0dHWhra4tdBhHVAoab56jVamRmZuLevXtil0JE1czMzAw2Nja8xxWRxDHcPKc42FhZWcHQ0JB/CRJJgFqtxoMHD5CdnQ0AsLW1FbkiIqpJDDfPUCqVQrBp3Lix2OUQUTUyMDAAAGRnZ8PKyopTVEQSxgXFzyheY2NoaChyJURUE4r/bHM9HZG0MdyUglNRRNLEP9tE9QPDDREREUkKww3VK9999x3s7e2hpaWF2NjYKh9vyJAh6Nu3b5WPQ0RE1YcLiqneyM3NRXh4OGJiYvD+++/D1NS0ysdcsmQJ1Gp1NVRHRETVheGG6oTCwkLo6urW6DnS09NRVFSEXr16VdulwNURkIiIqHpxWopEERAQgPDwcIwdOxYWFhYICgpCTEwMXF1dYWRkBHt7e4wcORL379/X2G/VqlWwt7eHoaEh3nvvPcTExMDMzKzc8yUkJMDV1RUA0KJFC8hkMly9erXUaaWxY8ciICBAeP/TTz/B1dUVBgYGaNy4MQIDA5Gfnw+g5LRUQUEBRo8eDSsrK+jr6+PNN9/E0aNHhc8VCgVkMhnkcjm8vLxgaGiIjh074sKFC5X7BRIRUZkYbkg0iYmJ0NXVxcGDBxEXFwctLS0sXboUZ86cQWJiIn7//XdMnDhRaH/w4EEMHz4cY8aMwYkTJ9CtWzfMnTu3QucKDg7G3r17AQApKSnIyMiAvb19uftlZGRgwIAB+PTTT3Hu3DkoFAr069evzKmoiRMnYvPmzUhMTERaWhpatWqFoKAg3LlzR6PdV199hUWLFuHYsWNo0KABPv300wr1g4iIysdpKRJN69at8fXXXwvvHR0dhZ8dHBwwZ84cDB8+HN988w0AYNmyZejZsyfGjx8PAGjTpg0OHTqEX3/9tdxzFY+6AIClpSVsbGwqVGNGRgYeP36Mfv36oVmzZgAgjAA9Lz8/HytXrkRCQgJ69uwJ4MlI0549e7BmzRpMmDBBaDt37lz4+/sDACZPnoxevXrh0aNH0NfXr1BdRESl8Zzw3yofI3VBSJWP4bfMr0r7H/z8YJX258gNicbT01Pj/d69e9G1a1fY2dnBxMQEgwYNwr///osHDx4AAC5cuABvb2+NfZ5/X93c3d3RtWtXuLq64sMPP8SqVatw9+7dUttevnwZRUVF8PN7+odaR0cH3t7eOHfunEZbNzc34efi9T/FjwYgIqKqYbgh0RgZGQk/X716Fe+88w7c3NywefNmpKamYsWKFQCeLDauKVpaWiWmmJ69e622tjb27NmDnTt3wsXFBcuWLYOjoyOuXLlSpfPq6OgIPxffWE6lUlXpmERE9ATDDdUJqampUKlUWLRoETp06IA2bdrg5s2bGm0cHR01FucCKPG+siwtLZGRkaGx7cSJExrvZTIZ/Pz8MHPmTBw/fhy6urrYunVriWO1bNlSWENUrKioCEePHoWLi0uV6iQioorjmhuqE1q1aoWioiIsW7YMvXv3FhYZP+vzzz9H586dERMTg969e+P333/Hzp07q3RL/S5dumDBggX473//C19fX/zwww84ffo02rdvDwA4cuQI5HI5unfvDisrKxw5cgS3bt2Cs7NziWMZGRlhxIgRmDBhAszNzdG0aVN8/fXXePDgAYYOHfrSNRJR5dWVtSevovRZpa8rrJRGDat+jCrgyA3VCe7u7oiJicH8+fPRrl07rFu3DlFRURpt/Pz8EBcXh5iYGLi7uyMpKQnjxo2r0iLcoKAgTJs2DRMnTsQbb7yBvLw8hIQ8/QutYcOG+OOPP/D222+jTZs2mDp1KhYtWiQsGH5edHQ03n//fQwaNAivv/46Ll26hF27dqFRo0YvXSMREVWOTF3Pbq+am5sLU1NT5OTkoGFDzWT56NEjXLlyBc2bN+dVK6+IsLAwnD9/Hvv37xe7FHoF8M94/VFfR26qo99bTRZU+RgDqjhyU9rVUi/6/n4ep6XolbJw4UJ069YNRkZG2LlzJxITE4VLxYmIiACGG3rFpKSk4Ouvv0ZeXh5atGiBpUuXYtiwYQCAtm3b4tq1a6Xu9+2332LgwIG1WSoRkej3e6mvGG7olbJp06YyP9uxY4fGZdzPsra2rqmSiIiojmG4IckovoMwERHVb7xaioiIiCSF4YaIiIgkhdNSREQSVl8via4OUriZXX3FkRsiIiKSFIYbIiIikhTRw82KFSvg4OAAfX19+Pj4ICUl5YXtY2Nj4ejoCAMDA9jb22PcuHF49OhRLVVLREREdZ2oa242btyIiIgIxMXFwcfHB7GxsQgKCsKFCxdgZWVVov369esxefJkxMfHo2PHjrh48SKGDBkCmUyGmJiYGq21OuatK6Om57iTk5PxySef4Pr16zV6HqK64uz121A9LkL23fsY/7+fkZlXWOljVMdt6ZtOP1XlY7yKeDM7qk2ijtzExMQgLCwMoaGhcHFxQVxcHAwNDREfH19q+0OHDsHPzw8ff/wxHBwc0L17dwwYMKDc0R4qadu2bejdu7fYZRAREVU70cJNYWEhUlNTERgY+LQYLS0EBgbi8OHDpe7TsWNHpKamCmHm77//xo4dO/D222+XeZ6CggLk5uZqvKTm119/hZmZGZRKJQDgxIkTkMlkmDx5stBm2LBh+OSTT4T327dvx7vvvgvgye9o9OjRsLKygr6+Pt58800cPXq0djtBRERUTUQLN7dv34ZSqSxxW3xra2tkZmaWus/HH3+MWbNm4c0334SOjg5atmyJgIAAfPnll2WeJyoqCqampsLL3t6+WvtRF3Tq1Al5eXk4fvw4gCdTThYWFlAoFEKb5ORkBAQEAADOnDmD7OxsdOnSBQAwceJEbN68GYmJiUhLS0OrVq0QFBSEO3fu1HZXiIiIqkz0BcWVoVAoMG/ePHzzzTdIS0vDli1b8Ntvv2H27Nll7jNlyhTk5OQILymuMTE1NYWHh4cQZhQKBcaNG4fjx4/j/v37uHHjBi5dugR/f38AT6akgoKCoKuri/z8fKxcuRILFixAz5494eLiglWrVsHAwABr1qwRsVdEREQvR7RwY2FhAW1tbWRlZWlsz8rKgo2NTan7TJs2DYMGDcKwYcPg6uqK9957D/PmzUNUVBRUKlWp++jp6aFhw4YaLyny9/eHQqGAWq3G/v370a9fPzg7O+PAgQNITk5GkyZN0Lp1awBPwk3xlNTly5dRVFQEP7+ni/10dHTg7e2Nc+fOidIXIiKiqhAt3Ojq6sLT0xNyuVzYplKpIJfL4evrW+o+Dx48gJaWZsna2toAALVaXXPFvgICAgJw4MABnDx5Ejo6OnByckJAQAAUCgWSk5OFUZuMjAwcP34cvXr1ErliIiKimiHqtFRERARWrVqFxMREnDt3DiNGjEB+fj5CQ0MBACEhIZgyZYrQvnfv3li5ciU2bNiAK1euYM+ePZg2bRp69+4thJz6qnjdzeLFi4UgUxxuFAqFsN7ml19+QceOHWFubg4AaNmyJXR1dXHw4NPLLIuKinD06FG4uLjUej+IiIiqStT73AQHB+PWrVuYPn06MjMz4eHhgaSkJGGRcXp6usZIzdSpUyGTyTB16lTcuHEDlpaW6N27N+bOnStWF+qMRo0awc3NDevWrcPy5csBAJ07d0b//v1RVFQkBJ5nr5ICACMjI4wYMQITJkyAubk5mjZtiq+//hoPHjzA0KFDRekLERFRVYj+4Mzw8HCEh4eX+tmzV/sAQIMGDRAZGYnIyMhaqOzV4+/vjxMnTgijNObm5nBxcUFWVhYcHR2Rn58PuVyO2NhYjf2io6OhUqkwaNAg5OXlwcvLC7t27UKjRo1qvxNEEvcq3syOD5CkV43o4eZV8So8FTc2NrZEcDlx4oTw865du9C8eXO0atVKo42+vj6WLl2KpUuX1kKVRERENeuVuhScqsbY2Bjz588XuwwiIqIaxZGbeqR79+5il0BERFTjOHJDREREksKRG6J65uz121U+hou9RZWPcT7rfJX2d7J2qnINRCRNHLkhIiIiSWG4ISIiIklhuCEiIiJJYbghIiIiSWG4ISIiIklhuKGX5uDgoHFHZJlMhp9//lm0euipIUOGoG/fvpXer1vH1/Hf1XHVXxARUS3ipeAVVC3PVqmEptNP1fg5kpOT8cknn+D69es1fi4q6c6dO4iMjMTu3buRnp4OS0tL9O3bF7Nnz4apqWmZ+82YMQMzZ84ssd3Q0BD5+flVqmnjL7thYGhYpWMUu3r1Kpo3b47jx4/Dw8OjWo75IjNmzMDPP/+s8cgRIqqfGG7qsW3btqF3795il1Fv3bx5Ezdv3sTChQvh4uKCa9euYfjw4bh58yZ++umnMvcbP348hg8frrGta9eueOONN6pck3njqt+/prIKCwuhq6tb6+clIunitJQE/PrrrzAzM4NSqQTw5GGZMpkMkydPFtoMGzYMn3zyicZ+27dvx7vvvlvmcTdv3oy2bdtCT08PDg4OWLRoUaXqmjRpEtq0aQNDQ0O0aNEC06ZNQ1FRkfD5jBkz4OHhgfj4eDRt2hTGxsYYOXIklEolvv76a9jY2MDKygpz587VOG5MTAxcXV1hZGQEe3t7jBw5Evfv3y+3nujoaFhbW8PExARDhw7F5MmThRGF06dPQ0tLC7du3QLwZFRFS0sLH330kbD/nDlz8OabbwrvT58+jZ49e8LY2BjW1tYYNGgQbt9+eoO8gIAAjB49GhMnToS5uTlsbGwwY8YM4fN27dph8+bN6N27N1q2bIkuXbpg7ty5+OWXX/D48eMy+2FsbAwbGxvhlZWVhbNnz2Lo0KEl2s6cOROWlpZo2LAhhg8fjsLCwhf+jp6flmrb1BI//fg9RocNhmebpujZ2Ru/704SPr979y4GDhwIS0tLGBgYoHXr1li7di0AoHnz5gCA9u3bQyaTCU+rL54yi4uNQ2f3znjb720AgLONM/bu3KtRj3cbb2zdsFV4n3kzE18M/wIdnDrAyMgIXl5eOHLkCBISEjBz5kycPHkSMpkMMpkMCQkJL+wrEUkXw40EdOrUCXl5eTh+/DiAJ9NNFhYWUCgUQpvk5GThywUAzpw5g+zsbHTp0qXUY6ampqJ///746KOPcOrUKcyYMQPTpk2r1BeGiYkJEhIScPbsWSxZsgSrVq3C4sWLNdpcvnwZO3fuRFJSEn788UesWbMGvXr1wj///IPk5GTMnz8fU6dOxZEjR4R9tLS0sHTpUpw5cwaJiYn4/fffMXHixBfWsmnTJsyYMQPz5s3DsWPHYGtri2+++Ub4vG3btmjcuDGSk5MBAPv379d4D2j+Du/du4cuXbqgffv2OHbsGJKSkpCVlYX+/ftrnDcxMRFGRkY4cuQIvv76a8yaNQt79uwps86cnBw0bNgQDRpUfFB19erVaNOmDTp16qSxXS6X49y5c1AoFPjxxx+xZcuWUqezyrMydiGC3umDLbsV6PxWICaNGY47d+4AAKZNm4azZ89i586dOHfuHFauXAkLiyejPykpKQCAvXv3IiMjA1u2bNGo7cqlK1izcQ1Wfr+yQnXk5+cj5L0QZGVmYUXiCpw8eRITJ06ESqVCcHAwvvjiC7Rt2xYZGRnIyMhAcHBwpftKRNLAaSkJMDU1hYeHBxQKBby8vKBQKDBu3DjMnDkT9+/fR05ODi5dugR/f39hn23btiEoKKjM6YCYmBh07doV06ZNAwC0adMGZ8+exYIFCzBkyJAK1TV16lThZwcHB4wfPx4bNmzQCCIqlQrx8fEwMTGBi4sL3nrrLVy4cAE7duyAlpYWHB0dMX/+fOzbtw8+Pj4AgLFjx2ocd86cORg+fLhGWHlebGwshg4dKoxuzJkzB3v37sWjR48APFkM3blzZygUCnzwwQdQKBQIDQ3F6tWrcf78ebRs2RKHDh0Sal++fDnat2+PefPmCeeIj4+Hvb09Ll68iDZt2gAA3NzcEBkZCQBo3bo1li9fDrlcjm7dupWo8fbt25g9ezY+++yzCv1+AeDRo0dYt26dxihdMV1dXcTHx8PQ0BBt27bFrFmzMGHCBAz4bAy0tCr+75o+H36EXn36AQDGTPoKP6xdhZSUFPTo0QPp6elo3749vLy8ADz571HM0tISANC4cWPY2NhoHNPIyAizY2ZXajrqty2/4c6/d7ApaRPMGpmhlXUrtGrVSvjc2NgYDRo0KHEuIqp/OHIjEf7+/lAoFFCr1di/fz/69esHZ2dnHDhwAMnJyWjSpAlat24ttN+2bdsLp6TOnTsHPz8/jW1+fn7466+/hOmv8mzcuBF+fn6wsbGBsbExpk6divT0dI02Dg4OMDExEd5bW1vDxcVF48vX2toa2dnZwvu9e/eia9eusLOzg4mJCQYNGoR///0XDx48APDkS674Vbw25dy5c0I4Kubr66vxvvh3CDwZpenSpYsQeI4ePYqioiLhd3Ly5Ens27dP41xOTk+edXT58mXhmG5ubhrnsLW11ehLsdzcXPTq1QsuLi4aU1dt27YVjt+zZ88S+23duhV5eXkYPHhwic/c3d1h+MziYF9fX9y/fx+ZN2/g160/wcupmfBKPXK4xP7FHJ1chJ8NDY1gbGIi9GHEiBHYsGEDPDw8MHHiRBw6dKjM4zzL1dW10utszp0+B+d2zjBrZFap/Yio/uHIjUQEBAQgPj4eJ0+ehI6ODpycnBAQEACFQoG7d+9qjNpkZGTg+PHj6NWrV43Vc/jwYQwcOBAzZ85EUFAQTE1NsWHDhhLrdnR0dDTey2SyUrepVCoAT67AeeeddzBixAjMnTsX5ubmOHDgAIYOHYrCwkIYGhpqXC3TsGHDCtccEBCAsWPH4q+//sLZs2fx5ptv4vz588Lv0MvLSwgL9+/fR+/evTF//vwSx7G1tX1h/4r7UiwvLw89evSAiYkJtm7dqrHPjh07hHVKBgYGJc61evVqvPPOO7C2tq5wPwHgrW494Nr+deG9tY1tmW0blNKHgn+vo+DmGXRxb4qLR3YhSb4f8v2H0LXrUgwf/BGip09AQdYNAEBh9mUU3Hx6DOWDezDQ1vwdFB9XrVZrbHtc9HTtkb6+fqX6SET1F8ONRBSvu1m8eLEQZAICAhAdHY27d+/iiy++ENr+8ssv6NixI8zNzcs8nrOzMw4ePKix7eDBg2jTpg20tbXLrefQoUNo1qwZvvrqK2HbtWvXKtutElJTU6FSqbBo0SJhdGfTpk0abZ6dqijm7OyMI0eOICQkRNj2559/arRxdXVFo0aNMGfOHHh4eMDY2BgBAQGYP38+7t69q7Fm6fXXX8fmzZvh4OBQqfUxz8vNzUVQUBD09PSwffv2El/gzZo1K3PfK1euYN++fdi+fXupn588eRIPHz4UQtGff/75ZDFyEztoaWnByNj4pet+lmVjcwzq3weD+vfBKu9N+HLOIkRPnwDd/x+KlKqKjfSZNzbHraxbwvurf1/Fw4cPhfeOLo74af1PuHf3XqmjN7q6uhUeVSQiaeO0lEQ0atQIbm5uWLdunfAl3LlzZ6SlpeHixYsaIzflXSUFAF988QXkcjlmz56NixcvIjExEcuXL8f48eMrVE/r1q2Rnp6ODRs24PLly1i6dCm2bt1a/o7laNWqFYqKirBs2TL8/fff+P777xEXV/5N58aMGYP4+HisXbsWFy9eRGRkJM6cOaPRpnjdzbO/Qzc3NxQUFEAul2v8DkeNGoU7d+5gwIABOHr0KC5fvoxdu3YhNDS0wl+wubm56N69O/Lz87FmzRrk5uYiMzMTmZmZFTpGfHw8bG1tS52uAp5cYj106FCcPXsWO3bsQGRkJMLDwyu13qY8Mxcsxy+7fsflK+k4e+ESdu5NhlPrFgAAKwtzGOjrY/e+A8i6dRs5uXkvPJbPmz5YH78eZ0+dxekTpzFz4kyNUay333sbFlYWCA8NR1pKGv7++29s3rwZhw8/mVJzcHDAlStXcOLECdy+fRsFBQXV1k8ierUw3EiIv78/lEql8MVsbm4OFxcX2NjYwNHREcCTK07kcnm54eb111/Hpk2bsGHDBrRr1w7Tp0/HrFmzKryY+N1338W4ceMQHh4ODw8PHDp0SFicXBXu7u6IiYnB/Pnz0a5dO6xbtw5RUVHl7hccHIxp06Zh4sSJ8PT0xLVr1zBixIgS7Z7/HWppaaFz586QyWQaa5CaNGmCgwcPQqlUonv37nB1dcXYsWNhZmZW4fCQlpaGI0eO4NSpU2jVqhVsbW2FV3k3VlSpVEhISMCQIUPKHEnr2rUrWrdujc6dOyM4OBjvvvuuxnqe6qCro4NpUbHwCuyHwH6Doa2tjf9+swAA0KBBAyyaPRmrf/gfmr/eBR98+vkLjzVpxiTY2NlgUJ9BGD9yPEJHhELf4OlIlq6uLlZvWI3GjRvjPwP/A1dXV0RHRwv9f//999GjRw+89dZbsLS0xI8//litfSWiV4dM/fwkt8Tl5ubC1NRUuOT2WY8ePcKVK1fQvHlzyc7vb9myBVOnTsXZs2fFLkV09fWOtmev3y6/UTlaamdV+RhXKjC9+SJO1k6Van/2+m2oHhchO+MfLNp7GZl5L77nT2m2miyo9D7PG9Co4uvASnPw84PlN3qG54T/Vul8APtdFez3yymt3y/6/n4eR27qGWNj41IXwRIREUkFFxTXM927dxe7BCIiohrFkRuqt2bMmFHvpqSIiOoDhhsiIiKSFIYbIiIikhSGGyIiIpIUhhsiIiKSFIYbIiIikpQ6EW5WrFgBBwcH6Ovrw8fHBykpKWW2DQgIgEwmK/GqyYdAEgFPH6xZV71sfW2bWkK+a0f1F0REJBLRw83GjRsRERGByMhIpKWlwd3dHUFBQcjOzi61/ZYtW5CRkSG8Tp8+DW1tbXz44Ye1XPmrLzk5Gfb29mKXUWFFRUWYNGkSXF1dYWRkhCZNmiAkJAQ3b94sd9/SAvGGDRuEz4cMGVJqm7Zt29ZklwA8uTP2kCFD4OrqigYNGqBv374vbH/w4EE0aNAAHh4e1XJ+xbHT6BTQtVqOlXwoBfp27XAvJ7dajleeIUOGlPv7IqL6R/Sb+MXExCAsLAyhoaEAgLi4OPz222+Ij4/H5MmTS7R//knWGzZsgKGhYY2HG79lfuU3qkaVveX2y9i2bRt69+5d4+epLg8ePEBaWhqmTZsGd3d33L17F2PGjMG7776LY8eOlbv/2rVr0aNHD+G9mZmZ8POSJUsQHR0tvH/8+DHc3d1rJTQrlUoYGBhg9OjR2Lx58wvb3rt3DyEhIejatSuysqr+CAQAsLSyrpbjVEZhYRFgULXHLxARlUXUkZvCwkKkpqYiMDBQ2KalpYXAwEDhSb/lWbNmDT766CMYGRnVVJl13q+//gozMzPhSdInTpyATCbTCIfDhg3DJ598orFfeU8HT0hIQNOmTWFoaIj33nsPixYtEgJBTk4OtLW1hVChUqlgbm6ODh06CPv/8MMPGiND169fR//+/WFmZgZzc3P06dMHV69eFT4v/lf4woULYWtri8aNG2PUqFEoKioCAJiammLPnj3o378/HB0d0aFDByxfvhypqalIT08v9/dkZmYGGxsb4fXs88NMTU01Pjt27Bju3r0rhO5ijx8/Rnh4OExNTWFhYYFp06ahvMezxcTECKNN9vb2GDlyJO7fvy98bmRkhJUrVyIsLAw2NjYvPNbw4cPx8ccfw9fXt9TPX6a+Z6elblxPR9umltiz81cMCe4LzzZN8V5QAE6kHhXa3/znOvoNHgUbl44wb/UG2r/VB0nyP3D1+g0EffgpAMDGpSP07dph2NivAADdPhiCsV/Nxfjp0bBr9ybe+fgz3Ei/AWcbZ5w7fU44dm5OLpxtnJFy8OnU9F/n/8LwT4bDq5UXPFt64pM+nyD9ajpmzJiBxMREbNu2TRhpUygUL+wrEdUPooab27dvQ6lUwtpa81+O1tbWyMzMLHf/lJQUnD59GsOGDSuzTUFBAXJzczVeUtOpUyfk5eXh+PHjAJ5MN1lYWGj8RZ+cnCw86RoAzpw5g+zsbHTp0qXUYx45cgRDhw5FeHg4Tpw4gbfeegtz5swRPjc1NYWHh4dwjlOnTkEmk+H48ePCF3dycjL8/f0BPJlSCgoKgomJCfbv34+DBw/C2NgYPXr0QGHh0wcY7tu3D5cvX8a+ffuQmJiIhIQEJCQklNn3nJwcyGQyjVGYsowaNQoWFhbw9vZGfHz8C7/016xZg8DAQDRr1kxje2JiIho0aICUlBQsWbIEMTExWL169QvPq6WlhaVLl+LMmTNITEzE77//jokTJ5Zb7/PWrl2Lv//+G5GRkWW2eZn6SrN0wTyEfjYSm5P2waF5C0z4/D94/PgxAGDO1EkoKCzE3s0JOCbfgrlfjoORkSHsm9hgw6rFAIBTf/yKq8cVWDTracD+4X/boKurg30/f4/l0dMrVEdWRhYGvTcIunq6SPgpAT/t/gn9PuoH5WMlxo8fj/79+6NHjx7CNHXHjh0r3Vcikh7Rp6WqYs2aNXB1dYW3t3eZbaKiojBz5sxarKr2PRs0vLy8oFAoMG7cOMycORP3799HTk4OLl26JAQN4MmUVFBQEHR1dUs95pIlS9CjRw/hS7hNmzY4dOgQkpKShDYBAQFQKBQYP348FAoFunXrhvPnz+PAgQPo0aMHFAqFsP/GjRuhUqmwevVqyGQyAE++rM3MzKBQKIRnXjVq1AjLly+HtrY2nJyc0KtXL8jlcoSFhZWo8dGjR5g0aRIGDBhQ7hNiZ82ahS5dusDQ0BC7d+8WRk9Gjx5dou3Nmzexc+dOrF+/vsRn9vb2WLx4MWQyGRwdHXHq1CksXry41PqKPbvI18HBAXPmzMHw4cPxzTffAADOZ50XPs95mIP7Bfc1tgHA1b+vYsLECfh+2/e49O8l3L5/GwWPC3A+67zG07Ffpr7SDPlsJPy7PvlvMipiEvoEvon0q1fQolVrZNy8gf69AtDOuQ0AoEWzp6NzjcxMAQCWFuYwM9X8b9KqeTPMm/qF8D77Rvn/gFm/dj1MTEywKG4RdHR0AADNWzYH8OQhsAYGBigoKCh3xIuI6hdRR24sLCygra1dYu1AVlZWuX9Z5efnY8OGDRg6dOgL202ZMgU5OTnC6/r161Wuuy7y9/eHQqGAWq3G/v370a9fPzg7O+PAgQNITk5GkyZN0Lp1a6H9tm3bXjglde7cOfj4+Ghse34qxN/fHwcOHIBSqRRGhooDz82bN3Hp0iVhtOjkyZO4dOkSTExMYGxsDGNjY5ibm+PRo0e4fPmycMy2bdtCW/vpWgxbW9tSF5cXFRWhf//+UKvVWLlypbC9Z8+ewvGfXQw8bdo0+Pn5oX379pg0aRImTpyIBQsWlNr3xMREmJmZlbpQtUOHDkI4K/6d/PXXX1AqlZg3b55wbmNjY2GqbO/evejatSvs7OxgYmKCQYMG4d9//8WDBw9KPf/zlEolJoyYgPAJ4cIXe1kqUp+XUzN4OTXDzRv/lHmcNs5Pf3fFa3Lu/HsLADAwdBiil3yHgD6fYNbC5Th19kKF+tHezaVC7Z517vQ5ePp4CsGGiKgiRB250dXVhaenJ+RyufBFolKpIJfLER4e/sJ9//e//6GgoKDEOpLn6enpQU9Pr7pKrrMCAgIQHx+PkydPQkdHB05OTkLQuHv3rsaoTUZGBo4fP17ly+c7d+6MvLw8pKWl4Y8//sC8efNgY2OD6OhouLu7awSq+/fvw9PTE+vWrStxHEtLS+Hn57/EZDIZVCqVxrbiYHPt2jX8/vvvGqM2q1evxsOHD0s91rN8fHwwe/ZsFBQUaPz/oVarER8fj0GDBpU5qlWW4cOHo3///sL7Jk2a4OrVq3jnnXcwYsQIzJ07F+bm5jhw4ACGDh2KwsJCGBoalnvc/Pv5OH3yNM6dPoc5Xz6ZGlSpVFCr1Whn1w67d+8uc3qxtPr+yrgDALCyLvsfEA0aPP2roTgsqVRPpvE+GDAIH7/liiT5H9j7xyEsWL4a86dPwMhPB77w/EYGBhrvZVpPjvvs9ODjoscabfQN9EFEVFmiT0tFRERg8ODB8PLygre3N2JjY5Gfny8s5AwJCYGdnR2ioqI09luzZg369u2Lxo0bi1F2nVO87mbx4sVCkAkICEB0dDTu3r2LL754Oh3wyy+/oGPHjiWuPHuWs7Mzjhw5orHtzz//1HhvZmYGNzc3LF++XAhUVlZWCA4Oxq+//qoRqF5//XVs3LgRVlZW5U4hvUhxsPnrr7+wb9++Ev/97ezsKnScEydOoFGjRiWCb3JyMi5dulTmiGBpv5PWrVtDW1sb5ubmJX6nqampUKlUWLRoEbS0ngyUbtq0qUI1FjM2Mca2fds0tv2Y8COOHDyC2FWx8Hnj6QhbReor1LtdqfOXxt7OFmEhwQgLCcbUqMWIX/8TRn46ELr/P1AqlapyjgCYN37yu7qVdQtwfbLt3JlzGm0cnR3x86afUVRUVGpY1dXVFRbSExEVEz3cBAcH49atW5g+fToyMzPh4eGBpKQkYZFxenq68KVQ7MKFCzhw4AB2794tRsl1UqNGjeDm5oZ169Zh+fLlAJ6MrPTv3x9FRUUaQaO8q6QAYPTo0fDz88PChQvRp08f7Nq1S2O9TbGAgAAsW7YMH3zwAYAnl+o7Oztj48aNWLFihdBu4MCBWLBgAfr06YNZs2bhtddew7Vr17BlyxZMnDgRr732Wrl9LCoqwgcffIC0tDT8+uuvUCqVwsJzc3PzMkdafvnlF2RlZaFDhw7Q19fHnj17MG/ePAwOG4mz1zW/6GOWfgO39p7QMrUp8dmDgiJcvXYNg8NGoP/AwTh7+v+wbNkyLFq0qMyaW7VqhaKiIixbtgy9e/fGwYMHERcXV6LdpQuXUFRUhJx7Oci/ny9cQeTczhlaWlpo8//XtxRrbNEYenp6aOPcRuNKwfT0dEREROA///kP0tLSyq3vZUTN+ArBXT3RqkUz3MvJxR8Hj8KpVQsAQNPXmkAmk2HH3mT06NoJBvr6MDYqfXRK30Af7p7uWLV8FV5r+hr+vf0vlkYv1Wjz8acf44f4H/DF8C/w2eefwbihMU6mnoRbezc4WTvBwcEBu3btwoULF9C4cWOYmppyCouIxL+JHwCEh4fj2rVrKCgowJEjRzTWeigUihJXyzg6OkKtVqNbt261XGnd5u/vD6VSKaxzMTc3h4uLC2xsbODo6AjgyVoluVxebrjp0KEDVq1ahSVLlsDd3R27d+/G1KlTyz0n8CTwPL/N0NAQf/zxB5o2bSqsBxo6dCgePXpU4ZGcGzduYPv27fjnn3/g4eEBW1tb4XXo0KEy99PR0cGKFSvg6+sLDw8PfPvtt4iJicHIcRM02uXl5mLPzl/RL7js6ZV33++PgkeP8NG73TFn6iSMGTMGn332WZnt3d3dERMTg/nz56Ndu3ZYt25diVFIAPjPwP+gX2A/7Nu9DymHUtAvsB/6BfarwG9FU0hICB4+fAhvb2+MGjWq3PpehkqlxJiv5sAj4F28O3A4WrVohiXzpgEA7GytMe2LUZgWtRhN3f0x9qu5LzzW3MVzoXysxAdBHyB6ejRGT9Zc4N3IvBESfkrAg/wHCHkvBB90/wD/W/c/NNB58u+ysLAwODo6wsvLC5aWljh4sObvD0VEdZ9MXd5NMCQmNzcXpqamyMnJKfGl+ujRI1y5cgXNmzfXuAeKlGzZsgVTp07F2bNnK71vQkICxo4di3v37lV/YSJ4fmTmZbjYW1T5GM9fGVVZz14tVRHV0e+W2lW/geAV7ardxO9l+q16XITsjH+waO9lZOYVlr/Tc7aalL4IvTIGNHr5aVmg8jf49Jzw3yqdD2C/q4L9fjml9ftF39/PqxMjN1R7jI2NMX/+fLHLICIiqjGir7mh2lV8PxkiIiKp4sgNVdiQIUMkMyVFRETSxXBDREREksJwQ0RERJLCcFOKenYBGVE9ooZaDZR/i0EiepVxQfEzim/+9eDBAxg8d6t4otIU3DxT9YNU8ZJoqjhVUSEeK1XIefi4/MZE9MpiuHmGtrY2zMzMhAc1GhoaajyEkKRF9bioyscoUFd9DEClqtr/Y48eParc+ephv9VqNR4/vI97d/7Fgcv/ouAxx26IpIzh5jnFTyMv7UnUJC3Zd+9X+RhqWW6Vj3Fbq2qzw7K8yoWE+trvjFu5OHD5X+w8W/WbGBJR3cZw8xyZTAZbW1tYWVmhqKjq/8Klumv8/36u8jGWGcVX+RjTTY3Kb/QCP37yY6Xa19d+D/3vTxyxIaonGG7KoK2tDW2uhZC0l7n9/vMaIKPKx8jWrdptyiv7qJD62m8GG6L6g1dLERERkaQw3BAREZGkMNwQERGRpDDcEBERkaQw3BAREZGkMNwQERGRpDDcEBERkaQw3BAREZGkMNwQERGRpDDcEBERkaQw3BAREZGkMNwQERGRpDDcEBERkaQw3BAREZGkMNwQERGRpDDcEBERkaQw3BAREZGkMNwQERGRpIgeblasWAEHBwfo6+vDx8cHKSkpL2x/7949jBo1Cra2ttDT00ObNm2wY8eOWqqWiIiI6roGYp5848aNiIiIQFxcHHx8fBAbG4ugoCBcuHABVlZWJdoXFhaiW7dusLKywk8//QQ7Oztcu3YNZmZmtV88ERER1UmihpuYmBiEhYUhNDQUABAXF4fffvsN8fHxmDx5con28fHxuHPnDg4dOgQdHR0AgIODQ22WTERERHWcaNNShYWFSE1NRWBg4NNitLQQGBiIw4cPl7rP9u3b4evri1GjRsHa2hrt2rXDvHnzoFQqa6tsIiIiquNEG7m5ffs2lEolrK2tNbZbW1vj/Pnzpe7z999/4/fff8fAgQOxY8cOXLp0CSNHjkRRUREiIyNL3aegoAAFBQXC+9zc3OrrBBEREdU5oi8orgyVSgUrKyt899138PT0RHBwML766ivExcWVuU9UVBRMTU2Fl729fS1WTERERLVNtHBjYWEBbW1tZGVlaWzPysqCjY1NqfvY2tqiTZs20NbWFrY5OzsjMzMThYWFpe4zZcoU5OTkCK/r169XXyeIiIiozhEt3Ojq6sLT0xNyuVzYplKpIJfL4evrW+o+fn5+uHTpElQqlbDt4sWLsLW1ha6ubqn76OnpoWHDhhovIiIiki5Rp6UiIiKwatUqJCYm4ty5cxgxYgTy8/OFq6dCQkIwZcoUof2IESNw584djBkzBhcvXsRvv/2GefPmYdSoUWJ1gYiIiOoYUS8FDw4Oxq1btzB9+nRkZmbCw8MDSUlJwiLj9PR0aGk9zV/29vbYtWsXxo0bBzc3N9jZ2WHMmDGYNGmSWF2QBM8J/63yMbaaLKjyMZpOP1XlYxAREYkabgAgPDwc4eHhpX6mUChKbPP19cWff/5Zw1URERHRq+qVulqKiIiIqDwMN0RERCQpok9LERXzW+ZXpf0Pfn6wmiohIqJXGUduiIiISFIYboiIiEhSGG6IiIhIUhhuiIiISFIYboiIiEhSGG6IiIhIUhhuiIiISFIYboiIiEhSGG6IiIhIUhhuiIiISFIYboiIiEhSGG6IiIhIUhhuiIiISFIYboiIiEhSGG6IiIhIUhhuiIiISFIYboiIiEhSGG6IiIhIUhhuiIiISFIYboiIiEhSGG6IiIhIUhhuiIiISFIYboiIiEhSGG6IiIhIUhhuiIiISFIYboiIiEhSGG6IiIhIUhhuiIiISFLqRLhZsWIFHBwcoK+vDx8fH6SkpJTZNiEhATKZTOOlr69fi9USERFRXSZ6uNm4cSMiIiIQGRmJtLQ0uLu7IygoCNnZ2WXu07BhQ2RkZAiva9eu1WLFREREVJeJHm5iYmIQFhaG0NBQuLi4IC4uDoaGhoiPjy9zH5lMBhsbG+FlbW1dixUTERFRXSZquCksLERqaioCAwOFbVpaWggMDMThw4fL3O/+/fto1qwZ7O3t0adPH5w5c6bMtgUFBcjNzdV4ERERkXSJGm5u374NpVJZYuTF2toamZmZpe7j6OiI+Ph4bNu2DT/88ANUKhU6duyIf/75p9T2UVFRMDU1FV729vbV3g8iIiKqO0SflqosX19fhISEwMPDA/7+/tiyZQssLS3x7bffltp+ypQpyMnJEV7Xr1+v5YqJiIioNjUQ8+QWFhbQ1tZGVlaWxvasrCzY2NhU6Bg6Ojpo3749Ll26VOrnenp60NPTq3KtRERE9GoQdeRGV1cXnp6ekMvlwjaVSgW5XA5fX98KHUOpVOLUqVOwtbWtqTKJiIjoFSLqyA0AREREYPDgwfDy8oK3tzdiY2ORn5+P0NBQAEBISAjs7OwQFRUFAJg1axY6dOiAVq1a4d69e1iwYAGuXbuGYcOGidkNIiIiqiNEDzfBwcG4desWpk+fjszMTHh4eCApKUlYZJyeng4tracDTHfv3kVYWBgyMzPRqFEjeHp64tChQ3BxcRGrC0RERFSHiB5uACA8PBzh4eGlfqZQKDTeL168GIsXL66FqoiIiOhV9MpdLUVERET0Igw3REREJCkMN0RERCQpDDdEREQkKQw3REREJCkMN0RERCQpDDdEREQkKQw3REREJCkMN0RERCQpdeIOxXWF54T/VvkYqQtCqnwMv2V+Vdr/4OcHq1wDERHRq+qlRm66dOmCe/fuldiem5uLLl26VLUmIiIiopf2UuFGoVCgsLCwxPZHjx5h//79VS6KiIiI6GVValrq//7v/4Sfz549i8zMTOG9UqlEUlIS7Ozsqq86IiIiokqqVLjx8PCATCaDTCYrdfrJwMAAy5Ytq7biXkXps1yrfpBGDat+DCIionqqUuHmypUrUKvVaNGiBVJSUmBpaSl8pqurCysrK2hra1d7kUREREQVValw06xZMwCASqWqkWKIiIiIqqrC4Wb79u3o2bMndHR0sH379he2fffdd6tcGBEREdHLqHC46du3LzIzM2FlZYW+ffuW2U4mk0GpVFZHbURERESVVuFw8+xUFKeliIiIqK566TsUy+VyyOVyZGdna4QdmUyGNWvWVEtxRERERJX1UuFm5syZmDVrFry8vGBrawuZTFbddRERERG9lJcKN3FxcUhISMCgQYOqux4iIiKiKnmpxy8UFhaiY8eO1V0LERERUZW9VLgZNmwY1q9fX921EBEREVVZhaelIiIihJ9VKhW+++477N27F25ubtDR0dFoGxMTU30VEhEREVVChcPN8ePHNd57eHgAAE6fPq2xnYuLiYiISEwVDjf79u2ryTqIiIiIqsVLrbkhIiIiqqsYboiIiEhSGG6IiIhIUupEuFmxYgUcHBygr68PHx8fpKSkVGi/DRs2QCaTvfBBnkRERFS/iB5uNm7ciIiICERGRiItLQ3u7u4ICgpCdnb2C/e7evUqxo8fj06dOtVSpURERPQqED3cxMTEICwsDKGhoXBxcUFcXBwMDQ0RHx9f5j5KpRIDBw7EzJkz0aJFi1qsloiIiOo6UcNNYWEhUlNTERgYKGzT0tJCYGAgDh8+XOZ+s2bNgpWVFYYOHVobZRIREdEr5KUenFldbt++DaVSCWtra43t1tbWOH/+fKn7HDhwAGvWrMGJEycqdI6CggIUFBQI73Nzc1+6XiIiIqr7RJ+Wqoy8vDwMGjQIq1atgoWFRYX2iYqKgqmpqfCyt7ev4SqJiIhITKKO3FhYWEBbWxtZWVka27OysmBjY1Oi/eXLl3H16lX07t1b2KZSqQAADRo0wIULF9CyZUuNfaZMmaLxXKzc3FwGHCIiIgkTNdzo6urC09MTcrlcuJxbpVJBLpcjPDy8RHsnJyecOnVKY9vUqVORl5eHJUuWlBpa9PT0oKenVyP1ExERUd0jargBnjxtfPDgwfDy8oK3tzdiY2ORn5+P0NBQAEBISAjs7OwQFRUFfX19tGvXTmN/MzMzACixnYiIiOon0cNNcHAwbt26henTpyMzMxMeHh5ISkoSFhmnp6dDS+uVWhpEREREIhI93ABAeHh4qdNQAKBQKF64b0JCQvUXRERERK8sDokQERGRpDDcEBERkaQw3BAREZGkMNwQERGRpDDcEBERkaQw3BAREZGkMNwQERGRpDDcEBERkaQw3BAREZGkMNwQERGRpDDcEBERkaQw3BAREZGkMNwQERGRpDDcEBERkaQw3BAREZGkMNwQERGRpDDcEBERkaQw3BAREZGkMNwQERGRpDDcEBERkaQw3BAREZGkMNwQERGRpDDcEBERkaQw3BAREZGkMNwQERGRpDDcEBERkaQw3BAREZGkMNwQERGRpDDcEBERkaQw3BAREZGkMNwQERGRpNSJcLNixQo4ODhAX18fPj4+SElJKbPtli1b4OXlBTMzMxgZGcHDwwPff/99LVZLREREdZno4Wbjxo2IiIhAZGQk0tLS4O7ujqCgIGRnZ5fa3tzcHF999RUOHz6M//u//0NoaChCQ0Oxa9euWq6ciIiI6iLRw01MTAzCwsIQGhoKFxcXxMXFwdDQEPHx8aW2DwgIwHvvvQdnZ2e0bNkSY8aMgZubGw4cOFDLlRMREVFdJGq4KSwsRGpqKgIDA4VtWlpaCAwMxOHDh8vdX61WQy6X48KFC+jcuXOpbQoKCpCbm6vxIiIiIukSNdzcvn0bSqUS1tbWGtutra2RmZlZ5n45OTkwNjaGrq4uevXqhWXLlqFbt26lto2KioKpqanwsre3r9Y+EBERUd0i+rTUyzAxMcGJEydw9OhRzJ07FxEREVAoFKW2nTJlCnJycoTX9evXa7dYIiIiqlUNxDy5hYUFtLW1kZWVpbE9KysLNjY2Ze6npaWFVq1aAQA8PDxw7tw5REVFISAgoERbPT096OnpVWvdREREVHeJOnKjq6sLT09PyOVyYZtKpYJcLoevr2+Fj6NSqVBQUFATJRIREdErRtSRGwCIiIjA4MGD4eXlBW9vb8TGxiI/Px+hoaEAgJCQENjZ2SEqKgrAkzU0Xl5eaNmyJQoKCrBjxw58//33WLlypZjdICIiojpC9HATHByMW7duYfr06cjMzISHhweSkpKERcbp6enQ0no6wJSfn4+RI0fin3/+gYGBAZycnPDDDz8gODhYrC4QERFRHSJ6uAGA8PBwhIeHl/rZ8wuF58yZgzlz5tRCVURERPQqeiWvliIiIiIqC8MNERERSQrDDREREUkKww0RERFJCsMNERERSQrDDREREUkKww0RERFJCsMNERERSQrDDREREUkKww0RERFJCsMNERERSQrDDREREUkKww0RERFJCsMNERERSQrDDREREUkKww0RERFJCsMNERERSQrDDREREUkKww0RERFJCsMNERERSQrDDREREUkKww0RERFJCsMNERERSQrDDREREUkKww0RERFJCsMNERERSQrDDREREUkKww0RERFJCsMNERERSQrDDREREUlKnQg3K1asgIODA/T19eHj44OUlJQy265atQqdOnVCo0aN0KhRIwQGBr6wPREREdUvooebjRs3IiIiApGRkUhLS4O7uzuCgoKQnZ1danuFQoEBAwZg3759OHz4MOzt7dG9e3fcuHGjlisnIiKiukj0cBMTE4OwsDCEhobCxcUFcXFxMDQ0RHx8fKnt161bh5EjR8LDwwNOTk5YvXo1VCoV5HJ5LVdOREREdZGo4aawsBCpqakIDAwUtmlpaSEwMBCHDx+u0DEePHiAoqIimJub11SZRERE9AppIObJb9++DaVSCWtra43t1tbWOH/+fIWOMWnSJDRp0kQjID2roKAABQUFwvvc3NyXL5iIiIjqPNGnpaoiOjoaGzZswNatW6Gvr19qm6ioKJiamgove3v7Wq6SiIiIapOo4cbCwgLa2trIysrS2J6VlQUbG5sX7rtw4UJER0dj9+7dcHNzK7PdlClTkJOTI7yuX79eLbUTERFR3SRquNHV1YWnp6fGYuDixcG+vr5l7vf1119j9uzZSEpKgpeX1wvPoaenh4YNG2q8iIiISLpEXXMDABERERg8eDC8vLzg7e2N2NhY5OfnIzQ0FAAQEhICOzs7REVFAQDmz5+P6dOnY/369XBwcEBmZiYAwNjYGMbGxqL1g4iIiOoG0cNNcHAwbt26henTpyMzMxMeHh5ISkoSFhmnp6dDS+vpANPKlStRWFiIDz74QOM4kZGRmDFjRm2WTkRERHWQ6OEGAMLDwxEeHl7qZwqFQuP91atXa74gIiIiemW90ldLERERET2P4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCRF9HCzYsUKODg4QF9fHz4+PkhJSSmz7ZkzZ/D+++/DwcEBMpkMsbGxtVcoERERvRJEDTcbN25EREQEIiMjkZaWBnd3dwQFBSE7O7vU9g8ePECLFi0QHR0NGxubWq6WiIiIXgWihpuYmBiEhYUhNDQULi4uiIuLg6GhIeLj40tt/8Ybb2DBggX46KOPoKenV8vVEhER0atAtHBTWFiI1NRUBAYGPi1GSwuBgYE4fPhwtZ2noKAAubm5Gi8iIiKSLtHCze3bt6FUKmFtba2x3draGpmZmdV2nqioKJiamgove3v7ajs2ERER1T2iLyiuaVOmTEFOTo7wun79utglERERUQ1qINaJLSwsoK2tjaysLI3tWVlZ1bpYWE9Pj+tziIiI6hHRRm50dXXh6ekJuVwubFOpVJDL5fD19RWrLCIiInrFiTZyAwAREREYPHgwvLy84O3tjdjYWOTn5yM0NBQAEBISAjs7O0RFRQF4sgj57Nmzws83btzAiRMnYGxsjFatWonWDyIiIqo7RA03wcHBuHXrFqZPn47MzEx4eHggKSlJWGScnp4OLa2ng0s3b95E+/bthfcLFy7EwoUL4e/vD4VCUdvlExERUR0kargBgPDwcISHh5f62fOBxcHBAWq1uhaqIiIioleV5K+WIiIiovqF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJIXhhoiIiCSF4YaIiIgkheGGiIiIJKVOhJsVK1bAwcEB+vr68PHxQUpKygvb/+9//4OTkxP09fXh6uqKHTt21FKlREREVNeJHm42btyIiIgIREZGIi0tDe7u7ggKCkJ2dnap7Q8dOoQBAwZg6NChOH78OPr27Yu+ffvi9OnTtVw5ERER1UWih5uYmBiEhYUhNDQULi4uiIuLg6GhIeLj40ttv2TJEvTo0QMTJkyAs7MzZs+ejddffx3Lly+v5cqJiIioLhI13BQWFiI1NRWBgYHCNi0tLQQGBuLw4cOl7nP48GGN9gAQFBRUZnsiIiKqXxqIefLbt29DqVTC2tpaY7u1tTXOnz9f6j6ZmZmlts/MzCy1fUFBAQoKCoT3OTk5AIDc3NwSbZUFDytVf2nydJRVPsbjh4+rtH9pfXsR9vvlsd8vj/1+Oex3xbDfL6+u9rt4m1qtLnd/UcNNbYiKisLMmTNLbLe3t6+R87WrkaNWjukk01o/J/stHva79rDf4mG/a09d73deXh5MTV/8exE13FhYWEBbWxtZWVka27OysmBjY1PqPjY2NpVqP2XKFERERAjvVSoV7ty5g8aNG0Mmk1WxB5WTm5sLe3t7XL9+HQ0bNqzVc4uJ/Wa/6wP2m/2uD8Tst1qtRl5eHpo0aVJuW1HDja6uLjw9PSGXy9G3b18AT8KHXC5HeHh4qfv4+vpCLpdj7NixwrY9e/bA19e31PZ6enrQ09PT2GZmZlYd5b+0hg0b1qs/DMXY7/qF/a5f2O/6Rax+lzdiU0z0aamIiAgMHjwYXl5e8Pb2RmxsLPLz8xEaGgoACAkJgZ2dHaKiogAAY8aMgb+/PxYtWoRevXphw4YNOHbsGL777jsxu0FERER1hOjhJjg4GLdu3cL06dORmZkJDw8PJCUlCYuG09PToaX19KKujh07Yv369Zg6dSq+/PJLtG7dGj///DPatasLs4REREQkNtHDDQCEh4eXOQ2lUChKbPvwww/x4Ycf1nBV1U9PTw+RkZElpsmkjv1mv+sD9pv9rg9elX7L1BW5poqIiIjoFSH6HYqJiIiIqhPDDREREUkKww0RERFJCsNNLfjjjz/Qu3dvNGnSBDKZDD///LPYJdWKqKgovPHGGzAxMYGVlRX69u2LCxcuiF1WjVu5ciXc3NyE+0D4+vpi586dYpdV66KjoyGTyTTuSSVFM2bMgEwm03g5OTmJXVatuHHjBj755BM0btwYBgYGcHV1xbFjx8Quq0Y5ODiU+O8tk8kwatQosUurUUqlEtOmTUPz5s1hYGCAli1bYvbs2RV6FIIY6sTVUlKXn58Pd3d3fPrpp+jXr5/Y5dSa5ORkjBo1Cm+88QYeP36ML7/8Et27d8fZs2dhZGQkdnk15rXXXkN0dDRat24NtVqNxMRE9OnTB8ePH0fbtm3FLq9WHD16FN9++y3c3NzELqVWtG3bFnv37hXeN2gg/b9a7969Cz8/P7z11lvYuXMnLC0t8ddff6FRo0Zil1ajjh49CqXy6bOXTp8+jW7dur2SV/BWxvz587Fy5UokJiaibdu2OHbsGEJDQ2FqaorRo0eLXV4J0v8TWAf07NkTPXv2FLuMWpeUlKTxPiEhAVZWVkhNTUXnzp1Fqqrm9e7dW+P93LlzsXLlSvz555/1Itzcv38fAwcOxKpVqzBnzhyxy6kVDRo0KPMRMFI1f/582NvbY+3atcK25s2bi1hR7bC0tNR4Hx0djZYtW8Lf31+kimrHoUOH0KdPH/Tq1QvAkxGsH3/8ESkpKSJXVjpOS1GtKX4iu7m5uciV1B6lUokNGzYgPz+/zEeESM2oUaPQq1cvBAYGil1Krfnrr7/QpEkTtGjRAgMHDkR6errYJdW47du3w8vLCx9++CGsrKzQvn17rFq1SuyyalVhYSF++OEHfPrpp7X+rMLa1rFjR8jlcly8eBEAcPLkSRw4cKDO/sOdIzdUK1QqFcaOHQs/P796cTfpU6dOwdfXF48ePYKxsTG2bt0KFxcXscuqcRs2bEBaWhqOHj0qdim1xsfHBwkJCXB0dERGRgZmzpyJTp064fTp0zAxMRG7vBrz999/Y+XKlYiIiMCXX36Jo0ePYvTo0dDV1cXgwYPFLq9W/Pzzz7h37x6GDBkidik1bvLkycjNzYWTkxO0tbWhVCoxd+5cDBw4UOzSSsVwQ7Vi1KhROH36NA4cOCB2KbXC0dERJ06cQE5ODn766ScMHjwYycnJkg44169fx5gxY7Bnzx7o6+uLXU6tefZfrm5ubvDx8UGzZs2wadMmDB06VMTKapZKpYKXlxfmzZsHAGjfvj1Onz6NuLi4ehNu1qxZg549e1boKdWvuk2bNmHdunVYv3492rZtixMnTmDs2LFo0qRJnfzvzXBDNS48PBy//vor/vjjD7z22mtil1MrdHV10apVKwCAp6cnjh49iiVLluDbb78VubKak5qaiuzsbLz++uvCNqVSiT/++APLly9HQUEBtLW1RaywdpiZmaFNmza4dOmS2KXUKFtb2xJh3dnZGZs3bxapotp17do17N27F1u2bBG7lFoxYcIETJ48GR999BEAwNXVFdeuXUNUVBTDDdUvarUan3/+ObZu3QqFQlEvFhuWRaVSoaCgQOwyalTXrl1x6tQpjW2hoaFwcnLCpEmT6kWwAZ4sqL58+TIGDRokdik1ys/Pr8StHS5evIhmzZqJVFHtWrt2LaysrIQFtlL34MEDjYdYA4C2tjZUKpVIFb0Yw00tuH//vsa/4q5cuYITJ07A3NwcTZs2FbGymjVq1CisX78e27Ztg4mJCTIzMwEApqamMDAwELm6mjNlyhT07NkTTZs2RV5eHtavXw+FQoFdu3aJXVqNMjExKbGeysjICI0bN5b0Oqvx48ejd+/eaNasGW7evInIyEhoa2tjwIABYpdWo8aNG4eOHTti3rx56N+/P1JSUvDdd9/hu+++E7u0GqdSqbB27VoMHjy4Xlz2Dzy5CnTu3Llo2rQp2rZti+PHjyMmJgaffvqp2KWVTk01bt++fWoAJV6DBw8Wu7QaVVqfAajXrl0rdmk16tNPP1U3a9ZMraurq7a0tFR37dpVvXv3brHLEoW/v796zJgxYpdRo4KDg9W2trZqXV1dtZ2dnTo4OFh96dIlscuqFb/88ou6Xbt2aj09PbWTk5P6u+++E7ukWrFr1y41APWFCxfELqXW5ObmqseMGaNu2rSpWl9fX92iRQv1V199pS4oKBC7tFLxqeBEREQkKbzPDREREUkKww0RERFJCsMNERERSQrDDREREUkKww0RERFJCsMNERERSQrDDREREUkKww0RERFJCsMNEdVrDg4OiI2NFbsMIqpGDDdEVGcEBARg7NixYpdBRK84hhsiIiKSFIYbIqoThgwZguTkZCxZsgQymQwymQxXr15FcnIyvL29oaenB1tbW0yePBmPHz8W9gsICEB4eDjCw8NhamoKCwsLTJs2DS/72LzVq1fDzMwMcrm8urpGRLWM4YaI6oQlS5bA19cXYWFhyMjIQEZGBnR0dPD222/jjTfewMmTJ7Fy5UqsWbMGc+bM0dg3MTERDRo0QEpKCpYsWYKYmBisXr260jV8/fXXmDx5Mnbv3o2uXbtWV9eIqJY1ELsAIiIAMDU1ha6uLgwNDWFjYwMA+Oqrr2Bvb4/ly5dDJpPByckJN2/exKRJkzB9+nRoaT3595m9vT0WL14MmUwGR0dHnDp1CosXL0ZYWFiFzz9p0iR8//33SE5ORtu2bWukj0RUOzhyQ0R11rlz5+Dr6wuZTCZs8/Pzw/379/HPP/8I2zp06KDRxtfXF3/99ReUSmWFzrNo0SKsWrUKBw4cYLAhkgCGGyKq9zp16gSlUolNmzaJXQoRVQOGGyKqM3R1dTVGW5ydnXH48GGNxcEHDx6EiYkJXnvtNWHbkSNHNI7z559/onXr1tDW1q7Qeb29vbFz507MmzcPCxcurGIviEhsDDdEVGc4ODjgyJEjuHr1Km7fvo2RI0fi+vXr+Pzzz3H+/Hls27YNkZGRiIiIENbbAEB6ejoiIiJw4cIF/Pjjj1i2bBnGjBlTqXN37NgRO3bswMyZM3lTP6JXHBcUE1GdMX78eAwePBguLi54+PAhrly5gh07dmDChAlwd3eHubk5hg4diqlTp2rsFxISgocPH8Lb2xva2toYM2YMPvvss0qf/80338Rvv/2Gt99+G9ra2vj888+rq2tEVItk6pe9GQQRUR0QEBAADw8PjrYQkYDTUkRERCQpDDdEJFn79++HsbFxmS8ikiZOSxGRZD18+BA3btwo8/NWrVrVYjVEVFsYboiIiEhSOC1FREREksJwQ0RERJLCcENERESSwnBDREREksJwQ0RERJLCcENERESSwnBDREREksJwQ0RERJLy/wC5uMUcg44/UAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "sns.barplot(x='top_k', y='hit', hue='rag_fusion', data=hit_stat_df, errorbar=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6a7d154-5f31-467c-bec9-5e710940cfad",
   "metadata": {},
   "source": [
    "# 预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "6857cb9d-e1de-491f-9b57-3d64b40932d7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T15:23:06.587963Z",
     "iopub.status.busy": "2024-08-18T15:23:06.587831Z",
     "iopub.status.idle": "2024-08-18T15:23:06.590850Z",
     "shell.execute_reply": "2024-08-18T15:23:06.590470Z",
     "shell.execute_reply.started": "2024-08-18T15:23:06.587949Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain.llms import Ollama\n",
    "\n",
    "ollama_llm = Ollama(\n",
    "    model='qwen2:7b-instruct',\n",
    "    base_url='http://localhost:11434'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "6bc495a3-2203-4c6a-aeba-a3b9b9691e0d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T15:23:06.591363Z",
     "iopub.status.busy": "2024-08-18T15:23:06.591242Z",
     "iopub.status.idle": "2024-08-18T15:23:09.460744Z",
     "shell.execute_reply": "2024-08-18T15:23:09.460177Z",
     "shell.execute_reply.started": "2024-08-18T15:23:06.591351Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'我是阿里云开发的一款超大规模语言模型，我叫通义千问。作为一个AI助手，我的目标是帮助用户获得准确、有用的信息，解决他们的问题和困惑。我会不断学习和进步，努力提供更好的服务。如果您有任何问题或需要帮助，请随时告诉我，我会尽力提供支持。'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ollama_llm.invoke('你是谁')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "b6bde30d-c569-47e9-ad82-fa08a13bed0d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T15:23:09.461493Z",
     "iopub.status.busy": "2024-08-18T15:23:09.461328Z",
     "iopub.status.idle": "2024-08-18T15:23:09.465029Z",
     "shell.execute_reply": "2024-08-18T15:23:09.464475Z",
     "shell.execute_reply.started": "2024-08-18T15:23:09.461477Z"
    }
   },
   "outputs": [],
   "source": [
    "def rag(query_gen_llm, question, n_chunks=3):\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个金融分析师，擅长根据所获取的信息片段，对问题进行分析和推理。\n",
    "你的任务是根据所获取的信息片段（<<<<context>>><<<</context>>>之间的内容）回答问题。\n",
    "回答保持简洁，不必重复问题，不要添加描述性解释和与答案无关的任何内容。\n",
    "已知信息：\n",
    "<<<<context>>>\n",
    "{{knowledge}}\n",
    "<<<</context>>>\n",
    "\n",
    "问题：{{question}}\n",
    "请回答：\n",
    "\"\"\".strip()\n",
    "\n",
    "    # rag_fusion_chain = get_rag_fusion_chain(n_chunks, trunc=True)\n",
    "    # chunks = rag_fusion_chain.invoke({'question': question})\n",
    "    \n",
    "    chunks = retrieve_with_rrf(query_gen_llm, question, top_k=n_chunks)\n",
    "    prompt = prompt_tmpl.replace('{{knowledge}}', '\\n\\n'.join([pair[0].page_content for pair in chunks])).replace('{{question}}', question)\n",
    "\n",
    "    return ollama_llm(prompt), chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "5f116e5d-e156-44cc-af4a-a6478918bfd8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T15:24:29.927263Z",
     "iopub.status.busy": "2024-08-18T15:24:29.926447Z",
     "iopub.status.idle": "2024-08-18T15:24:33.005790Z",
     "shell.execute_reply": "2024-08-18T15:24:33.005223Z",
     "shell.execute_reply.started": "2024-08-18T15:24:29.927186Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:139: LangChainDeprecationWarning: The method `BaseLLM.__call__` was deprecated in langchain-core 0.1.7 and will be removed in 0.3.0. Use invoke instead.\n",
      "  warn_deprecated(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023年10月，美国ISM制造业PMI指数较上个月大幅下降了2.3个百分点。\n"
     ]
    }
   ],
   "source": [
    "print(rag(llm, '2023年10月美国ISM制造业PMI指数较上月有何变化？')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "7940483e-b265-47b3-a9a7-5cd5173695b1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T15:24:35.301678Z",
     "iopub.status.busy": "2024-08-18T15:24:35.300887Z",
     "iopub.status.idle": "2024-08-18T15:24:35.324652Z",
     "shell.execute_reply": "2024-08-18T15:24:35.322387Z",
     "shell.execute_reply.started": "2024-08-18T15:24:35.301603Z"
    }
   },
   "outputs": [],
   "source": [
    "prediction_df = qa_df[qa_df['dataset'] == 'test'][['uuid', 'question', 'qa_type', 'answer']].rename(columns={'answer': 'ref_answer'})\n",
    "\n",
    "def predict(query_gen_llm, prediction_df, n_chunks):\n",
    "    prediction_df = prediction_df.copy()\n",
    "    answer_dict = {}\n",
    "\n",
    "    for idx, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        uuid = row['uuid']\n",
    "        question = row['question']\n",
    "        answer, chunks = rag(query_gen_llm, question, n_chunks=n_chunks)\n",
    "        assert len(chunks) <= n_chunks\n",
    "        answer_dict[question] = {\n",
    "            'uuid': uuid,\n",
    "            'ref_answer': row['ref_answer'],\n",
    "            'gen_answer': answer,\n",
    "            'chunks': chunks\n",
    "        }\n",
    "    prediction_df.loc[:, 'gen_answer'] = prediction_df['question'].apply(lambda q: answer_dict[q]['gen_answer'])\n",
    "    prediction_df.loc[:, 'chunks'] = prediction_df['question'].apply(lambda q: answer_dict[q]['chunks'])\n",
    "\n",
    "    return prediction_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "14f5086f-8812-4235-862f-6f0505ab443d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T15:24:36.704479Z",
     "iopub.status.busy": "2024-08-18T15:24:36.703679Z",
     "iopub.status.idle": "2024-08-18T15:38:14.660072Z",
     "shell.execute_reply": "2024-08-18T15:38:14.657635Z",
     "shell.execute_reply.started": "2024-08-18T15:24:36.704404Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2824a5da66d54edb81e1c531565b86d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a309279322ab42a6809624d6698cef30",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_chunks = 3\n",
    "\n",
    "pred_df_dict = {}\n",
    "for llm_name, llm_model in zip(['ollama-qwen2-7b-instruct', 'qwen2-57b-a14b-instruct'], [llm, qwen2_14b_llm]):\n",
    "    pred_df_dict[llm_name] = predict(llm_model, prediction_df, n_chunks=n_chunks)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ae9f9b8-8fbf-41be-a8ec-ed05831d2a91",
   "metadata": {},
   "source": [
    "# 评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "7c4912ad-9a7a-4187-a0fd-2985086294e5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T15:38:21.874301Z",
     "iopub.status.busy": "2024-08-18T15:38:21.873512Z",
     "iopub.status.idle": "2024-08-18T15:38:21.888983Z",
     "shell.execute_reply": "2024-08-18T15:38:21.888528Z",
     "shell.execute_reply.started": "2024-08-18T15:38:21.874226Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain_openai import ChatOpenAI\n",
    "import time\n",
    "\n",
    "judge_llm = ChatOpenAI(\n",
    "    api_key=os.environ['LLM_API_KEY'],\n",
    "    base_url=os.environ['LLM_BASE_URL'],\n",
    "    model_name='qwen2-72b-instruct',\n",
    "    temperature=0\n",
    ")\n",
    "\n",
    "def evaluate(prediction_df):\n",
    "    \"\"\"\n",
    "    对预测结果进行打分\n",
    "    :param prediction_df: 预测结果，需要包含问题，参考答案，生成的答案，列名分别为question, ref_answer, gen_answer\n",
    "    :return 打分模型原始返回结果\n",
    "    \"\"\"\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个经济学博士，现在我有一系列问题，有一个助手已经对这些问题进行了回答，你需要参照参考答案，评价这个助手的回答是否正确，仅回复“是”或“否”即可，不要带其他描述性内容或无关信息。\n",
    "问题：\n",
    "<question>\n",
    "{{question}}\n",
    "</question>\n",
    "\n",
    "参考答案：\n",
    "<ref_answer>\n",
    "{{ref_answer}}\n",
    "</ref_answer>\n",
    "\n",
    "助手回答：\n",
    "<gen_answer>\n",
    "{{gen_answer}}\n",
    "</gen_answer>\n",
    "请评价：\n",
    "    \"\"\"\n",
    "    results = []\n",
    "\n",
    "    for _, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        question = row['question']\n",
    "        ref_answer = row['ref_answer']\n",
    "        gen_answer = row['gen_answer']\n",
    "\n",
    "        prompt = prompt_tmpl.replace('{{question}}', question).replace('{{ref_answer}}', str(ref_answer)).replace('{{gen_answer}}', gen_answer).strip()\n",
    "        result = judge_llm.invoke(prompt).content\n",
    "        results.append(result)\n",
    "\n",
    "        time.sleep(1)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "d234cf90-b30a-4097-98c7-b59365b08824",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T15:38:30.720685Z",
     "iopub.status.busy": "2024-08-18T15:38:30.719531Z",
     "iopub.status.idle": "2024-08-18T15:43:35.904592Z",
     "shell.execute_reply": "2024-08-18T15:43:35.902218Z",
     "shell.execute_reply.started": "2024-08-18T15:38:30.720625Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a9d72d993e74103b7e7d81fc77b8b85",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ollama-qwen2-7b-instruct: ['是' '否']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2336286/2492278847.py:4: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n",
      "  pred_df.loc[:, 'score'] = pred_df['raw_score'].replace({'是': 1, '否': 0})\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "706c772ba422400c8bc7164df84a3dd6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qwen2-57b-a14b-instruct: ['是' '否']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2336286/2492278847.py:4: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n",
      "  pred_df.loc[:, 'score'] = pred_df['raw_score'].replace({'是': 1, '否': 0})\n"
     ]
    }
   ],
   "source": [
    "for model_name, pred_df in pred_df_dict.items():\n",
    "    pred_df['raw_score'] = evaluate(pred_df)\n",
    "    print(f\"{model_name}: {pred_df['raw_score'].unique()}\")\n",
    "    pred_df.loc[:, 'score'] = pred_df['raw_score'].replace({'是': 1, '否': 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "c91c4d40-4d54-4971-81c5-2eee450b688b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T15:44:15.169494Z",
     "iopub.status.busy": "2024-08-18T15:44:15.168712Z",
     "iopub.status.idle": "2024-08-18T15:44:15.184317Z",
     "shell.execute_reply": "2024-08-18T15:44:15.182023Z",
     "shell.execute_reply.started": "2024-08-18T15:44:15.169421Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.69"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_df_dict['ollama-qwen2-7b-instruct']['score'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "b185f59e-6cf5-4dcb-8aee-a685e846a85d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T15:44:15.626023Z",
     "iopub.status.busy": "2024-08-18T15:44:15.625805Z",
     "iopub.status.idle": "2024-08-18T15:44:15.630123Z",
     "shell.execute_reply": "2024-08-18T15:44:15.629589Z",
     "shell.execute_reply.started": "2024-08-18T15:44:15.626005Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.69"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_df_dict['qwen2-57b-a14b-instruct']['score'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db5ae164-0cb7-47bc-bb49-618ab557d47c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-18T15:44:49.375748Z",
     "iopub.status.busy": "2024-08-18T15:44:49.374940Z"
    }
   },
   "outputs": [],
   "source": [
    "for model_name, pred_df in pred_df_dict.items():\n",
    "    pred_df.to_excel(os.path.join(expr_dir, f'prediction_{model_name}.xlsx'), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3eb95a60-e189-4738-9a1c-5105cf085f98",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
